Coverage for documents/filters.py: 58%
498 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-10-10 09:07 +0000
1from __future__ import annotations
3import functools
4import inspect
5import json
6import logging
7import operator
8from contextlib import contextmanager
9from decimal import Decimal
10from decimal import InvalidOperation
11from typing import TYPE_CHECKING
12from typing import Any
14from django.contrib.contenttypes.models import ContentType
15from django.db.models import Case
16from django.db.models import CharField
17from django.db.models import Count
18from django.db.models import Exists
19from django.db.models import IntegerField
20from django.db.models import OuterRef
21from django.db.models import Q
22from django.db.models import Subquery
23from django.db.models import Sum
24from django.db.models import Value
25from django.db.models import When
26from django.db.models.functions import Cast
27from django.db.models.functions import NullIf
28from django.utils.translation import gettext_lazy as _
29from django_filters import DateFilter
30from django_filters.rest_framework import BooleanFilter
31from django_filters.rest_framework import CharFilter
32from django_filters.rest_framework import DateTimeFilter
33from django_filters.rest_framework import Filter
34from django_filters.rest_framework import FilterSet
35from django_filters.rest_framework import MultipleChoiceFilter
36from drf_spectacular.utils import extend_schema_field
37from guardian.utils import get_group_obj_perms_model
38from guardian.utils import get_user_obj_perms_model
39from rest_framework import serializers
40from rest_framework.filters import BaseFilterBackend
41from rest_framework.filters import OrderingFilter
43from documents.models import Correspondent
44from documents.models import CustomField
45from documents.models import CustomFieldInstance
46from documents.models import Document
47from documents.models import DocumentType
48from documents.models import PaperlessTask
49from documents.models import ShareLink
50from documents.models import ShareLinkBundle
51from documents.models import StoragePath
52from documents.models import Tag
53from documents.permissions import permitted_document_ids
54from documents.permissions import permitted_object_ids
55from documents.versioning import annotate_effective_content
57if TYPE_CHECKING: 57 ↛ 58line 57 didn't jump to line 58 because the condition on line 57 was never true
58 from collections.abc import Callable
60CHAR_KWARGS = ["istartswith", "iendswith", "icontains", "iexact"]
61ID_KWARGS = ["in", "exact"]
62INT_KWARGS = ["exact", "gt", "gte", "lt", "lte", "isnull"]
63DATE_KWARGS = [
64 "year",
65 "month",
66 "day",
67 "gt",
68 "gte",
69 "lt",
70 "lte",
71]
72DATETIME_KWARGS = [
73 "year",
74 "month",
75 "day",
76 "date__gt",
77 "date__gte",
78 "gt",
79 "gte",
80 "date__lt",
81 "date__lte",
82 "lt",
83 "lte",
84]
86CUSTOM_FIELD_QUERY_MAX_DEPTH = 10
87CUSTOM_FIELD_QUERY_MAX_ATOMS = 20
89logger = logging.getLogger("paperless.api")
92class CorrespondentFilterSet(FilterSet):
93 class Meta:
94 model = Correspondent
95 fields = {
96 "id": ID_KWARGS,
97 "name": CHAR_KWARGS,
98 }
101class TagFilterSet(FilterSet):
102 class Meta:
103 model = Tag
104 fields = {
105 "id": ID_KWARGS,
106 "name": CHAR_KWARGS,
107 }
109 is_root = BooleanFilter(
110 label="Is root tag",
111 field_name="tn_parent",
112 lookup_expr="isnull",
113 )
116class DocumentTypeFilterSet(FilterSet):
117 class Meta:
118 model = DocumentType
119 fields = {
120 "id": ID_KWARGS,
121 "name": CHAR_KWARGS,
122 }
125class StoragePathFilterSet(FilterSet):
126 class Meta:
127 model = StoragePath
128 fields = {
129 "id": ID_KWARGS,
130 "name": CHAR_KWARGS,
131 "path": CHAR_KWARGS,
132 }
135class ObjectFilter(Filter):
136 def __init__(self, *, exclude=False, in_list=False, field_name="") -> None:
137 super().__init__()
138 self.exclude = exclude
139 self.in_list = in_list
140 self.field_name = field_name
142 def filter(self, qs, value):
143 if not value:
144 return qs
146 try:
147 object_ids = [int(x) for x in value.split(",")]
148 except ValueError:
149 return qs
151 if self.in_list:
152 qs = qs.filter(**{f"{self.field_name}__id__in": object_ids}).distinct()
153 else:
154 for obj_id in object_ids:
155 if self.exclude:
156 qs = qs.exclude(**{f"{self.field_name}__id": obj_id})
157 else:
158 qs = qs.filter(**{f"{self.field_name}__id": obj_id})
160 return qs
163@extend_schema_field(serializers.BooleanField)
164class InboxFilter(Filter):
165 def filter(self, qs, value):
166 if value == "true":
167 # A document can have more than one tag flagged as an inbox tag
168 # (nothing enforces uniqueness), so this join can multiply rows.
169 return qs.filter(tags__is_inbox_tag=True).distinct()
170 elif value == "false":
171 return qs.exclude(tags__is_inbox_tag=True)
172 else:
173 return qs
176@extend_schema_field(serializers.CharField)
177class TitleContentFilter(Filter):
178 # Deprecated but retained for existing saved views. UI uses Tantivy-backed `text` / `title_search` params.
179 def filter(self, qs: Any, value: Any) -> Any:
180 value = value.strip() if isinstance(value, str) else value
181 if value:
182 logger.warning(
183 "Deprecated document filter parameter 'title_content' used; use `text` instead.",
184 )
185 return annotate_effective_content(qs).filter(
186 Q(title__icontains=value) | Q(effective_content__icontains=value),
187 )
188 else:
189 return qs
192@extend_schema_field(serializers.CharField)
193class EffectiveContentFilter(Filter):
194 def filter(self, qs: Any, value: Any) -> Any:
195 value = value.strip() if isinstance(value, str) else value
196 if not value:
197 return qs
198 return annotate_effective_content(qs).filter(
199 **{f"effective_content__{self.lookup_expr}": value},
200 )
203@extend_schema_field(serializers.BooleanField)
204class SharedByUser(Filter):
205 def filter(self, qs, value):
206 ctype = ContentType.objects.get_for_model(self.model)
207 UserObjectPermission = get_user_obj_perms_model()
208 GroupObjectPermission = get_group_obj_perms_model()
209 # see https://github.com/paperless-ngx/paperless-ngx/issues/5392, we limit subqueries
210 # to 1 because Postgres doesn't like returning > 1 row, but all we care about is > 0
211 return (
212 qs.filter(
213 owner_id=value,
214 )
215 .annotate(
216 num_shared_users=Count(
217 UserObjectPermission.objects.filter(
218 content_type=ctype,
219 object_pk=Cast(OuterRef("pk"), CharField()),
220 ).values("user_id")[:1],
221 ),
222 )
223 .annotate(
224 num_shared_groups=Count(
225 GroupObjectPermission.objects.filter(
226 content_type=ctype,
227 object_pk=Cast(OuterRef("pk"), CharField()),
228 ).values("group_id")[:1],
229 ),
230 )
231 .filter(
232 Q(num_shared_users__gt=0) | Q(num_shared_groups__gt=0),
233 )
234 if value is not None
235 else qs
236 )
239class CustomFieldFilterSet(FilterSet):
240 class Meta:
241 model = CustomField
242 fields = {
243 "id": ID_KWARGS,
244 "name": CHAR_KWARGS,
245 }
248@extend_schema_field(serializers.CharField)
249class CustomFieldsFilter(Filter):
250 def filter(self, qs, value):
251 value = value.strip() if isinstance(value, str) else value
252 if value:
253 logger.warning(
254 "Deprecated document filter parameter 'custom_fields__icontains' used; use `custom_field_query` or advanced Tantivy field syntax instead.",
255 )
256 fields_with_matching_selects = CustomField.objects.filter(
257 extra_data__icontains=value,
258 )
259 option_ids = []
260 if fields_with_matching_selects.count() > 0: 260 ↛ 261line 260 didn't jump to line 261 because the condition on line 260 was never true
261 for field in fields_with_matching_selects:
262 options = field.extra_data.get("select_options", [])
263 for _, option in enumerate(options):
264 if option.get("label").lower().find(value.lower()) != -1:
265 option_ids.extend([option.get("id")])
266 # A document with multiple custom field instances can match more
267 # than one of these OR-ed branches (or the same branch via
268 # different fields), each via its own join to custom_fields --
269 # dedupe explicitly rather than relying on the caller to.
270 return (
271 qs.filter(custom_fields__field__name__icontains=value)
272 | qs.filter(custom_fields__value_text__icontains=value)
273 | qs.filter(custom_fields__value_bool__icontains=value)
274 | qs.filter(custom_fields__value_int__icontains=value)
275 | qs.filter(custom_fields__value_float__icontains=value)
276 | qs.filter(custom_fields__value_date__icontains=value)
277 | qs.filter(custom_fields__value_url__icontains=value)
278 | qs.filter(custom_fields__value_monetary__icontains=value)
279 | qs.filter(custom_fields__value_document_ids__icontains=value)
280 | qs.filter(custom_fields__value_select__in=option_ids)
281 | qs.filter(custom_fields__value_long_text__icontains=value)
282 ).distinct()
283 else:
284 return qs
287class MimeTypeFilter(Filter):
288 def filter(self, qs, value):
289 value = value.strip() if isinstance(value, str) else value
290 if value:
291 return qs.filter(mime_type__icontains=value)
292 else:
293 return qs
296class MonetaryAmountField(serializers.Field):
297 """
298 Accepts either a plain decimal string ("100", "100.00") or a currency-prefixed
299 string ("USD100.00") and returns the numeric amount as a Decimal.
301 Mirrors the logic of the value_monetary_amount generated field: if the value
302 starts with a non-digit, the first 3 characters are treated as a currency code
303 (ISO 4217) and stripped before parsing. This preserves backwards compatibility
304 with saved views that stored a currency-prefixed string as the filter value.
305 """
307 default_error_messages = {"invalid": "A valid number is required."}
309 def to_internal_value(self, data):
310 if not isinstance(data, str | int | float):
311 self.fail("invalid")
312 value = str(data).strip()
313 if value and not value[0].isdigit() and value[0] != "-":
314 value = value[3:] # strip 3-char ISO 4217 currency code
315 try:
316 return Decimal(value)
317 except InvalidOperation:
318 self.fail("invalid")
320 def to_representation(self, value):
321 return str(value)
324class SelectField(serializers.CharField):
325 def __init__(self, custom_field: CustomField) -> None:
326 self._options = custom_field.extra_data["select_options"]
327 super().__init__(max_length=16)
329 def to_internal_value(self, data):
330 # If the supplied value is the option label instead of the ID
331 try:
332 data = next(
333 option.get("id")
334 for option in self._options
335 if option.get("label") == data
336 )
337 except StopIteration:
338 pass
339 return super().to_internal_value(data)
342def handle_validation_prefix(func: Callable):
343 """
344 Catch ValidationErrors raised by the wrapped function
345 and add a prefix to the exception detail to track what causes the exception,
346 similar to nested serializers.
347 """
349 def wrapper(*args, validation_prefix=None, **kwargs):
350 try:
351 return func(*args, **kwargs)
352 except serializers.ValidationError as e:
353 raise serializers.ValidationError({validation_prefix: e.detail})
355 # Update the signature to include the validation_prefix argument
356 old_sig = inspect.signature(func)
357 new_param = inspect.Parameter("validation_prefix", inspect.Parameter.KEYWORD_ONLY)
358 new_sig = old_sig.replace(parameters=[*old_sig.parameters.values(), new_param])
360 # Apply functools.wraps and manually set the new signature
361 functools.update_wrapper(wrapper, func)
362 wrapper.__signature__ = new_sig
364 return wrapper
367class CustomFieldQueryParser:
368 EXPR_BY_CATEGORY = {
369 "basic": ["exact", "in", "isnull", "exists"],
370 "string": [
371 "icontains",
372 "istartswith",
373 "iendswith",
374 ],
375 "arithmetic": [
376 "gt",
377 "gte",
378 "lt",
379 "lte",
380 "range",
381 ],
382 "containment": ["contains"],
383 }
385 SUPPORTED_EXPR_CATEGORIES = {
386 CustomField.FieldDataType.STRING: ("basic", "string"),
387 CustomField.FieldDataType.URL: ("basic", "string"),
388 CustomField.FieldDataType.DATE: ("basic", "arithmetic"),
389 CustomField.FieldDataType.BOOL: ("basic",),
390 CustomField.FieldDataType.INT: ("basic", "arithmetic"),
391 CustomField.FieldDataType.FLOAT: ("basic", "arithmetic"),
392 CustomField.FieldDataType.MONETARY: ("basic", "string", "arithmetic"),
393 CustomField.FieldDataType.DOCUMENTLINK: ("basic", "containment"),
394 CustomField.FieldDataType.SELECT: ("basic",),
395 CustomField.FieldDataType.LONG_TEXT: ("basic", "string"),
396 }
398 DATE_COMPONENTS = [
399 "year",
400 "iso_year",
401 "month",
402 "day",
403 "week",
404 "week_day",
405 "iso_week_day",
406 "quarter",
407 ]
409 def __init__(
410 self,
411 validation_prefix,
412 max_query_depth=10,
413 max_atom_count=20,
414 ) -> None:
415 """
416 A helper class that parses the query string into a `django.db.models.Q` for filtering
417 documents based on custom field values.
419 The syntax of the query expression is illustrated with the below pseudo code rules:
420 1. parse([`custom_field`, "exists", true]):
421 matches documents with Q(custom_fields__field=`custom_field`)
422 2. parse([`custom_field`, "exists", false]):
423 matches documents with ~Q(custom_fields__field=`custom_field`)
424 3. parse([`custom_field`, `op`, `value`]):
425 matches documents with
426 Q(custom_fields__field=`custom_field`, custom_fields__value_`type`__`op`= `value`)
427 4. parse(["AND", [`q0`, `q1`, ..., `qn`]])
428 -> parse(`q0`) & parse(`q1`) & ... & parse(`qn`)
429 5. parse(["OR", [`q0`, `q1`, ..., `qn`]])
430 -> parse(`q0`) | parse(`q1`) | ... | parse(`qn`)
431 6. parse(["NOT", `q`])
432 -> ~parse(`q`)
434 Args:
435 validation_prefix: Used to generate the ValidationError message.
436 max_query_depth: Limits the maximum nesting depth of queries.
437 max_atom_count: Limits the maximum number of atoms (i.e., rule 1, 2, 3) in the query.
439 `max_query_depth` and `max_atom_count` can be set to guard against generating arbitrarily
440 complex SQL queries.
441 """
442 self._custom_fields: dict[int | str, CustomField] = {}
443 self._validation_prefix = validation_prefix
444 # Dummy ModelSerializer used to convert a Django models.Field to serializers.Field.
445 self._model_serializer = serializers.ModelSerializer()
446 # Used for sanity check
447 self._max_query_depth = max_query_depth
448 self._max_atom_count = max_atom_count
449 self._current_depth = 0
450 self._atom_count = 0
451 # The set of annotations that we need to apply to the queryset
452 self._annotations = {}
454 def parse(self, query: str) -> tuple[Q, dict[str, Count]]:
455 """
456 Parses the query string into a `django.db.models.Q`
457 and a set of annotations to be applied to the queryset.
458 """
459 try:
460 expr = json.loads(query)
461 except json.JSONDecodeError:
462 raise serializers.ValidationError(
463 {self._validation_prefix: [_("Value must be valid JSON.")]},
464 )
465 return (
466 self._parse_expr(expr, validation_prefix=self._validation_prefix),
467 self._annotations,
468 )
470 @handle_validation_prefix
471 def _parse_expr(self, expr) -> Q:
472 """
473 Applies rule (1, 2, 3) or (4, 5, 6) based on the length of the expr.
474 """
475 with self._track_query_depth():
476 if isinstance(expr, list | tuple): 476 ↛ 477line 476 didn't jump to line 477 because the condition on line 476 was never true
477 if len(expr) == 2:
478 return self._parse_logical_expr(*expr)
479 elif len(expr) == 3:
480 return self._parse_atom(*expr)
481 raise serializers.ValidationError(
482 [_("Invalid custom field query expression")],
483 )
485 @handle_validation_prefix
486 def _parse_expr_list(self, exprs) -> list[Q]:
487 """
488 Handles [`q0`, `q1`, ..., `qn`] in rule 4 & 5.
489 """
490 if not isinstance(exprs, list | tuple) or not exprs:
491 raise serializers.ValidationError(
492 [_("Invalid expression list. Must be nonempty.")],
493 )
494 return [
495 self._parse_expr(expr, validation_prefix=i) for i, expr in enumerate(exprs)
496 ]
498 def _parse_logical_expr(self, op, args) -> Q:
499 """
500 Handles rule 4, 5, 6.
501 """
502 op_lower = op.lower()
504 if op_lower == "not":
505 return ~self._parse_expr(args, validation_prefix=1)
507 if op_lower == "and":
508 op_func = operator.and_
509 elif op_lower == "or":
510 op_func = operator.or_
511 else:
512 raise serializers.ValidationError(
513 {"0": [_("Invalid logical operator {op!r}").format(op=op)]},
514 )
516 qs = self._parse_expr_list(args, validation_prefix="1")
517 return functools.reduce(op_func, qs)
519 def _parse_atom(self, id_or_name, op, value) -> Q:
520 """
521 Handles rule 1, 2, 3.
522 """
523 # Guard against queries with too many conditions.
524 self._atom_count += 1
525 if self._atom_count > self._max_atom_count:
526 raise serializers.ValidationError(
527 [_("Maximum number of query conditions exceeded.")],
528 )
530 custom_field = self._get_custom_field(id_or_name, validation_prefix="0")
531 op = self._validate_atom_op(custom_field, op, validation_prefix="1")
532 value = self._validate_atom_value(
533 custom_field,
534 op,
535 value,
536 validation_prefix="2",
537 )
539 # Needed because not all DB backends support Array __contains
540 if (
541 custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
542 and op == "contains"
543 ):
544 return self._parse_atom_doc_link_contains(custom_field, value)
546 value_field_name = CustomFieldInstance.get_value_field_name(
547 custom_field.data_type,
548 )
549 if custom_field.data_type == CustomField.FieldDataType.MONETARY and (
550 op in self.EXPR_BY_CATEGORY["arithmetic"] or op in {"exact", "in"}
551 ):
552 value_field_name = "value_monetary_amount"
553 has_field = Q(custom_fields__field=custom_field)
555 # We need to use an annotation here because different atoms
556 # might be referring to different instances of custom fields.
557 annotation_name = f"_custom_field_filter_{len(self._annotations)}"
559 # Our special exists operator.
560 if op == "exists":
561 annotation = Count("custom_fields", filter=has_field)
562 # A Document should have > 0 match if it has this field, or 0 if doesn't.
563 query_op = "gt" if value else "exact"
564 query = Q(**{f"{annotation_name}__{query_op}": 0})
565 else:
566 # Check if 1) custom field name matches, and 2) value satisfies condition
567 field_filter = has_field & Q(
568 **{f"custom_fields__{value_field_name}__{op}": value},
569 )
570 # Annotate how many matching custom fields each document has
571 annotation = Count("custom_fields", filter=field_filter)
572 # Filter document by count
573 query = Q(**{f"{annotation_name}__gt": 0})
575 self._annotations[annotation_name] = annotation
576 return query
578 @handle_validation_prefix
579 def _get_custom_field(self, id_or_name):
580 """Get the CustomField instance by id or name."""
581 if id_or_name in self._custom_fields:
582 return self._custom_fields[id_or_name]
584 kwargs = (
585 {"id": id_or_name} if isinstance(id_or_name, int) else {"name": id_or_name}
586 )
587 try:
588 custom_field = CustomField.objects.get(**kwargs)
589 except CustomField.DoesNotExist:
590 raise serializers.ValidationError(
591 [_("{name!r} is not a valid custom field.").format(name=id_or_name)],
592 )
593 self._custom_fields[custom_field.id] = custom_field
594 self._custom_fields[custom_field.name] = custom_field
595 return custom_field
597 @staticmethod
598 def _split_op(full_op):
599 *prefix, op = str(full_op).rsplit("__", maxsplit=1)
600 prefix = prefix[0] if prefix else None
601 return prefix, op
603 @handle_validation_prefix
604 def _validate_atom_op(self, custom_field, raw_op):
605 """Check if the `op` is compatible with the type of the custom field."""
606 prefix, op = self._split_op(raw_op)
608 # Check if the operator is supported for the current data_type.
609 supported = False
610 for category in self.SUPPORTED_EXPR_CATEGORIES[custom_field.data_type]:
611 if op in self.EXPR_BY_CATEGORY[category]:
612 supported = True
613 break
615 # Check prefix
616 if prefix is not None:
617 if (
618 prefix in self.DATE_COMPONENTS
619 and custom_field.data_type == CustomField.FieldDataType.DATE
620 ):
621 pass # ok - e.g., "year__exact" for date field
622 else:
623 supported = False # anything else is invalid
625 if not supported:
626 raise serializers.ValidationError(
627 [
628 _("{data_type} does not support query expr {expr!r}.").format(
629 data_type=custom_field.data_type,
630 expr=raw_op,
631 ),
632 ],
633 )
635 return raw_op
637 def _get_serializer_field(self, custom_field, full_op):
638 """Return a serializers.Field for value validation."""
639 prefix, op = self._split_op(full_op)
640 field = None
642 if op in ("isnull", "exists"):
643 # `isnull` takes either True or False regardless of the data_type.
644 field = serializers.BooleanField()
645 elif (
646 custom_field.data_type == CustomField.FieldDataType.DATE
647 and prefix in self.DATE_COMPONENTS
648 ):
649 # DateField admits queries in the form of `year__exact`, etc. These take integers.
650 field = serializers.IntegerField()
651 elif custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK:
652 # We can be more specific here and make sure the value is a list.
653 field = serializers.ListField(child=serializers.IntegerField())
654 elif custom_field.data_type == CustomField.FieldDataType.SELECT:
655 # We use this custom field to permit SELECT option names.
656 field = SelectField(custom_field)
657 elif custom_field.data_type == CustomField.FieldDataType.URL:
658 # For URL fields we don't need to be strict about validation (e.g., for istartswith).
659 field = serializers.CharField()
660 elif custom_field.data_type == CustomField.FieldDataType.MONETARY and (
661 op in self.EXPR_BY_CATEGORY["arithmetic"] or op in {"exact", "in"}
662 ):
663 # These ops compare against value_monetary_amount (a DecimalField).
664 # MonetaryAmountField accepts both "100" and "USD100.00" for backwards
665 # compatibility with saved views that stored currency-prefixed values.
666 field = MonetaryAmountField()
667 else:
668 # The general case: inferred from the corresponding field in CustomFieldInstance.
669 value_field_name = CustomFieldInstance.get_value_field_name(
670 custom_field.data_type,
671 )
672 model_field = CustomFieldInstance._meta.get_field(value_field_name)
673 field_name = model_field.deconstruct()[0]
674 field_class, field_kwargs = self._model_serializer.build_standard_field(
675 field_name,
676 model_field,
677 )
678 field = field_class(**field_kwargs)
679 field.allow_null = False
681 # Need to set allow_blank manually because of the inconsistency in CustomFieldInstance validation.
682 # See https://github.com/paperless-ngx/paperless-ngx/issues/7361.
683 if isinstance(field, serializers.CharField):
684 field.allow_blank = True
686 if op == "in":
687 # `in` takes a list of values.
688 field = serializers.ListField(child=field, allow_empty=False)
689 elif op == "range":
690 # `range` takes a list of values, i.e., [start, end].
691 field = serializers.ListField(
692 child=field,
693 min_length=2,
694 max_length=2,
695 )
697 return field
699 @handle_validation_prefix
700 def _validate_atom_value(self, custom_field, op, value):
701 """Check if `value` is valid for the custom field and `op`. Returns the validated value."""
702 serializer_field = self._get_serializer_field(custom_field, op)
703 return serializer_field.run_validation(value)
705 def _parse_atom_doc_link_contains(self, custom_field, value) -> Q:
706 """
707 Handles document link `contains` in a way that is supported by all DB backends.
708 """
710 # If the value is an empty set,
711 # this is trivially true for any document with not null document links.
712 if not value:
713 return Q(
714 custom_fields__field=custom_field,
715 custom_fields__value_document_ids__isnull=False,
716 )
718 # First we look up reverse links from the requested documents.
719 # Scoped to this specific field (not just any document link field) and
720 # excluding unset instances, which have a null value_document_ids and
721 # are equivalent to having no reverse link at all.
722 links = CustomFieldInstance.objects.filter(
723 document_id__in=value,
724 field=custom_field,
725 value_document_ids__isnull=False,
726 )
728 # Check if any of the requested IDs are missing.
729 missing_ids = set(value) - set(link.document_id for link in links)
730 if missing_ids:
731 # The result should be an empty set in this case.
732 return Q(id__in=[])
734 # Take the intersection of the reverse links - this should be what we are looking for.
735 document_ids_we_want = functools.reduce(
736 operator.and_,
737 (set(link.value_document_ids) for link in links),
738 )
740 return Q(id__in=document_ids_we_want)
742 @contextmanager
743 def _track_query_depth(self):
744 # guard against queries that are too deeply nested
745 self._current_depth += 1
746 if self._current_depth > self._max_query_depth: 746 ↛ 747line 746 didn't jump to line 747 because the condition on line 746 was never true
747 raise serializers.ValidationError([_("Maximum nesting depth exceeded.")])
748 try:
749 yield
750 finally:
751 self._current_depth -= 1
754@extend_schema_field(serializers.CharField)
755class CustomFieldQueryFilter(Filter):
756 def __init__(self, validation_prefix) -> None:
757 """
758 A filter that filters documents based on custom field name and value.
760 Args:
761 validation_prefix: Used to generate the ValidationError message.
762 """
763 super().__init__()
764 self._validation_prefix = validation_prefix
766 def filter(self, qs, value):
767 if not value:
768 return qs
770 parser = CustomFieldQueryParser(
771 self._validation_prefix,
772 max_query_depth=CUSTOM_FIELD_QUERY_MAX_DEPTH,
773 max_atom_count=CUSTOM_FIELD_QUERY_MAX_ATOMS,
774 )
775 q, annotations = parser.parse(value)
777 # The Count(...) annotations above require a GROUP BY/HAVING to evaluate.
778 # Applying them directly to `qs` mixes that HAVING with `qs`'s existing
779 # joins (e.g. repeated tag joins from tags__id__all, the object-permission
780 # OR-filter), which some backends (e.g. MariaDB) fail to plan correctly,
781 # raising "Unknown column ... in 'HAVING'". Evaluating the annotation on
782 # an isolated queryset keeps the GROUP BY/HAVING self-contained.
783 matching_ids = Document.objects.annotate(**annotations).filter(q).values("pk")
784 return qs.filter(pk__in=matching_ids)
787class DocumentFilterSet(FilterSet):
788 has_duplicates = BooleanFilter(method="filter_has_duplicates")
790 def __init__(self, *args: Any, user: Any = None, **kwargs: Any) -> None:
791 super().__init__(*args, **kwargs)
792 self._user = user
794 is_tagged = BooleanFilter(
795 label="Is tagged",
796 field_name="tags",
797 lookup_expr="isnull",
798 exclude=True,
799 )
801 tags__id__all = ObjectFilter(field_name="tags")
803 tags__id__none = ObjectFilter(field_name="tags", exclude=True)
805 tags__id__in = ObjectFilter(field_name="tags", in_list=True)
807 correspondent__id__none = ObjectFilter(field_name="correspondent", exclude=True)
809 document_type__id__none = ObjectFilter(field_name="document_type", exclude=True)
811 storage_path__id__none = ObjectFilter(field_name="storage_path", exclude=True)
813 is_in_inbox = InboxFilter()
815 # Deprecated, but keep for now for existing saved views
816 title_content = TitleContentFilter()
818 content__istartswith = EffectiveContentFilter(lookup_expr="istartswith")
819 content__iendswith = EffectiveContentFilter(lookup_expr="iendswith")
820 content__icontains = EffectiveContentFilter(lookup_expr="icontains")
821 content__iexact = EffectiveContentFilter(lookup_expr="iexact")
823 owner__id__none = ObjectFilter(field_name="owner", exclude=True)
825 # Deprecated, UI no longer includes CF text-search mode, but keep for now for existing saved views
826 custom_fields__icontains = CustomFieldsFilter()
828 custom_fields__id__all = ObjectFilter(field_name="custom_fields__field")
830 custom_fields__id__none = ObjectFilter(
831 field_name="custom_fields__field",
832 exclude=True,
833 )
835 custom_fields__id__in = ObjectFilter(
836 field_name="custom_fields__field",
837 in_list=True,
838 )
840 has_custom_fields = BooleanFilter(
841 label="Has custom field",
842 field_name="custom_fields",
843 lookup_expr="isnull",
844 exclude=True,
845 )
847 custom_field_query = CustomFieldQueryFilter("custom_field_query")
849 shared_by__id = SharedByUser()
851 mime_type = MimeTypeFilter()
853 def filter_has_duplicates(self, queryset, name, value):
854 if value is None: 854 ↛ 855line 854 didn't jump to line 855 because the condition on line 854 was never true
855 return queryset
857 user = (
858 self._user
859 if self._user is not None
860 else getattr(self.request, "user", None)
861 )
862 queryset = queryset.alias(
863 nonempty_archive_checksum=NullIf("archive_checksum", Value("")),
864 )
866 visible_root_documents = Document.global_objects.filter(
867 root_document__isnull=True,
868 pk__in=permitted_document_ids(
869 user,
870 include_deleted=True,
871 ),
872 ).exclude(pk=OuterRef("pk"))
873 # see serialisers._get_viewable_duplicates().
874 matching_duplicates = visible_root_documents.filter(
875 Q(checksum=OuterRef("checksum"))
876 | Q(checksum=OuterRef("nonempty_archive_checksum"))
877 | Q(archive_checksum=OuterRef("checksum"))
878 | Q(archive_checksum=OuterRef("nonempty_archive_checksum")),
879 )
881 return queryset.alias(
882 has_visible_duplicates=Exists(matching_duplicates),
883 ).filter(has_visible_duplicates=value)
885 # Backwards compatibility
886 created__date__gt = DateFilter(field_name="created", lookup_expr="gt")
887 created__date__gte = DateFilter(field_name="created", lookup_expr="gte")
888 created__date__lt = DateFilter(field_name="created", lookup_expr="lt")
889 created__date__lte = DateFilter(field_name="created", lookup_expr="lte")
891 class Meta:
892 model = Document
893 fields = {
894 "id": ID_KWARGS,
895 "title": CHAR_KWARGS,
896 "archive_serial_number": INT_KWARGS,
897 "created": DATE_KWARGS,
898 "added": DATETIME_KWARGS,
899 "modified": DATETIME_KWARGS,
900 "original_filename": CHAR_KWARGS,
901 "checksum": CHAR_KWARGS,
902 "correspondent": ["isnull"],
903 "correspondent__id": ID_KWARGS,
904 "correspondent__name": CHAR_KWARGS,
905 "tags__id": ID_KWARGS,
906 "tags__name": CHAR_KWARGS,
907 "document_type": ["isnull"],
908 "document_type__id": ID_KWARGS,
909 "document_type__name": CHAR_KWARGS,
910 "storage_path": ["isnull"],
911 "storage_path__id": ID_KWARGS,
912 "storage_path__name": CHAR_KWARGS,
913 "owner": ["isnull"],
914 "owner__id": ID_KWARGS,
915 "custom_fields": ["icontains"],
916 }
919class ShareLinkFilterSet(FilterSet):
920 class Meta:
921 model = ShareLink
922 fields = {
923 "created": DATETIME_KWARGS,
924 "expiration": DATETIME_KWARGS,
925 }
928class ShareLinkBundleFilterSet(FilterSet):
929 documents = Filter(method="filter_documents")
931 class Meta:
932 model = ShareLinkBundle
933 fields = {
934 "created": DATETIME_KWARGS,
935 "expiration": DATETIME_KWARGS,
936 "status": ["exact"],
937 }
939 def filter_documents(self, queryset, name, value):
940 ids = []
941 if value: 941 ↛ 946line 941 didn't jump to line 946 because the condition on line 941 was always true
942 try:
943 ids = [int(item) for item in value.split(",") if item]
944 except ValueError:
945 return queryset.none()
946 if not ids: 946 ↛ 947line 946 didn't jump to line 947 because the condition on line 946 was never true
947 return queryset
948 return queryset.filter(documents__in=ids).distinct()
951class PaperlessTaskFilterSet(FilterSet):
952 name = CharFilter(
953 method="filter_name",
954 label="Name",
955 )
957 result = CharFilter(
958 method="filter_result",
959 label="Result",
960 )
962 task_type = MultipleChoiceFilter(
963 choices=PaperlessTask.TaskType.choices,
964 label="Task Type",
965 )
967 trigger_source = MultipleChoiceFilter(
968 choices=PaperlessTask.TriggerSource.choices,
969 label="Trigger Source",
970 )
972 status = MultipleChoiceFilter(
973 choices=PaperlessTask.Status.choices,
974 label="Status",
975 )
977 is_complete = BooleanFilter(
978 method="filter_is_complete",
979 label="Is Complete",
980 )
982 acknowledged = BooleanFilter(
983 label="Acknowledged",
984 field_name="acknowledged",
985 )
987 date_created_after = DateTimeFilter(
988 field_name="date_created",
989 lookup_expr="gte",
990 label="Created After",
991 )
993 date_created_before = DateTimeFilter(
994 field_name="date_created",
995 lookup_expr="lte",
996 label="Created Before",
997 )
999 class Meta:
1000 model = PaperlessTask
1001 fields = [
1002 "task_type",
1003 "trigger_source",
1004 "status",
1005 "acknowledged",
1006 "owner",
1007 "name",
1008 "result",
1009 ]
1011 def filter_name(self, queryset, name, value):
1012 if not value: 1012 ↛ 1013line 1012 didn't jump to line 1013 because the condition on line 1012 was never true
1013 return queryset
1015 matching_task_types = [
1016 task_type
1017 for task_type, label in PaperlessTask.TaskType.choices
1018 if value.lower() in str(label).lower()
1019 ]
1020 matching_trigger_sources = [
1021 trigger_source
1022 for trigger_source, label in PaperlessTask.TriggerSource.choices
1023 if value.lower() in str(label).lower()
1024 ]
1026 return queryset.filter(
1027 Q(input_data__filename__icontains=value)
1028 | Q(task_type__in=matching_task_types)
1029 | Q(trigger_source__in=matching_trigger_sources),
1030 )
1032 def filter_result(self, queryset, name, value):
1033 if not value: 1033 ↛ 1034line 1033 didn't jump to line 1034 because the condition on line 1033 was never true
1034 return queryset
1036 query = Q(result_data__reason__icontains=value) | Q(
1037 result_data__error_message__icontains=value,
1038 )
1040 try:
1041 numeric_value = int(value)
1042 except (TypeError, ValueError):
1043 pass
1044 else:
1045 query |= Q(result_data__document_id=numeric_value) | Q(
1046 result_data__duplicate_of=numeric_value,
1047 )
1049 if "duplicate" in value.lower(): 1049 ↛ 1050line 1049 didn't jump to line 1050 because the condition on line 1049 was never true
1050 query |= Q(result_data__duplicate_of__isnull=False)
1052 return queryset.filter(query)
1054 def filter_is_complete(self, queryset, name, value):
1055 if value:
1056 return queryset.filter(status__in=PaperlessTask.COMPLETE_STATUSES)
1057 return queryset.exclude(status__in=PaperlessTask.COMPLETE_STATUSES)
1060class PermittedObjectsFilter(BaseFilterBackend):
1061 """
1062 Filters a queryset down to objects the requesting user owns, are
1063 unowned, or (when ``include_granted`` is True) has an explicit
1064 user/group guardian permission on. Backed by ``permitted_object_ids``
1065 -- a single ``id__in`` subquery, not a join -- so it can't produce
1066 duplicate rows even when the base queryset already carries independent
1067 joins (e.g. multi-value ``tags__id__all`` filtering), and stays
1068 index-friendly at scale instead of falling back to guardian's
1069 varchar-cast join.
1071 Set ``include_granted = False`` on a subclass for endpoints that
1072 intentionally only show owned/unowned objects regardless of explicit
1073 shares (e.g. ``TrashView``).
1074 """
1076 include_granted: bool = True
1077 perm_codename: str | None = None
1079 def filter_queryset(self, request, queryset, view):
1080 # Before the superuser and owner-only paths, neither of which consults
1081 # permitted_object_ids. Scoped to authenticated users so anonymous
1082 # access (AnonymousUser.is_active is False) keeps its existing
1083 # unowned-only behaviour.
1084 if request.user.is_authenticated and not request.user.is_active: 1084 ↛ 1085line 1084 didn't jump to line 1085 because the condition on line 1084 was never true
1085 return queryset.none()
1086 if request.user.is_superuser:
1087 return queryset
1088 if not self.include_granted: 1088 ↛ 1089line 1088 didn't jump to line 1089 because the condition on line 1088 was never true
1089 return queryset.filter(Q(owner=request.user) | Q(owner__isnull=True))
1090 model = queryset.model
1091 perm = self.perm_codename or f"view_{model._meta.model_name}"
1092 return queryset.filter(
1093 id__in=permitted_object_ids(request.user, model, perm),
1094 )
1097class DocumentsOrderingFilter(OrderingFilter):
1098 field_name = "ordering"
1099 prefix = "custom_field_"
1101 def filter_queryset(self, request, queryset, view):
1102 param = request.query_params.get("ordering")
1103 if param and self.prefix in param: 1103 ↛ 1104line 1103 didn't jump to line 1104 because the condition on line 1103 was never true
1104 custom_field_id = int(param.split(self.prefix)[1])
1105 try:
1106 field = CustomField.objects.get(pk=custom_field_id)
1107 except CustomField.DoesNotExist:
1108 raise serializers.ValidationError(
1109 {self.prefix + str(custom_field_id): [_("Custom field not found")]},
1110 )
1112 annotation = None
1113 match field.data_type:
1114 case (
1115 CustomField.FieldDataType.STRING
1116 | CustomField.FieldDataType.LONG_TEXT
1117 ):
1118 annotation = Subquery(
1119 CustomFieldInstance.objects.filter(
1120 document_id=OuterRef("id"),
1121 field_id=custom_field_id,
1122 ).values("value_text")[:1],
1123 )
1124 case CustomField.FieldDataType.INT:
1125 annotation = Subquery(
1126 CustomFieldInstance.objects.filter(
1127 document_id=OuterRef("id"),
1128 field_id=custom_field_id,
1129 ).values("value_int")[:1],
1130 )
1131 case CustomField.FieldDataType.FLOAT:
1132 annotation = Subquery(
1133 CustomFieldInstance.objects.filter(
1134 document_id=OuterRef("id"),
1135 field_id=custom_field_id,
1136 ).values("value_float")[:1],
1137 )
1138 case CustomField.FieldDataType.DATE:
1139 annotation = Subquery(
1140 CustomFieldInstance.objects.filter(
1141 document_id=OuterRef("id"),
1142 field_id=custom_field_id,
1143 ).values("value_date")[:1],
1144 )
1145 case CustomField.FieldDataType.MONETARY:
1146 annotation = Subquery(
1147 CustomFieldInstance.objects.filter(
1148 document_id=OuterRef("id"),
1149 field_id=custom_field_id,
1150 ).values("value_monetary_amount")[:1],
1151 )
1152 case CustomField.FieldDataType.SELECT:
1153 # Select options are a little more complicated since the value is the id of the option, not
1154 # the label. Additionally, to support sqlite we can't use StringAgg, so we need to create a
1155 # case statement for each option, setting the value to the index of the option in a list
1156 # sorted by label, and then summing the results to give a single value for the annotation
1158 select_options = sorted(
1159 field.extra_data.get("select_options", []),
1160 key=lambda x: x.get("label"),
1161 )
1162 whens = [
1163 When(
1164 custom_fields__field_id=custom_field_id,
1165 custom_fields__value_select=option.get("id"),
1166 then=Value(idx, output_field=IntegerField()),
1167 )
1168 for idx, option in enumerate(select_options)
1169 ]
1170 whens.append(
1171 When(
1172 custom_fields__field_id=custom_field_id,
1173 custom_fields__value_select__isnull=True,
1174 then=Value(
1175 len(select_options),
1176 output_field=IntegerField(),
1177 ),
1178 ),
1179 )
1180 annotation = Sum(
1181 Case(
1182 *whens,
1183 default=Value(0),
1184 output_field=IntegerField(),
1185 ),
1186 )
1187 case CustomField.FieldDataType.DOCUMENTLINK:
1188 annotation = Subquery(
1189 CustomFieldInstance.objects.filter(
1190 document_id=OuterRef("id"),
1191 field_id=custom_field_id,
1192 ).values("value_document_ids")[:1],
1193 )
1194 case CustomField.FieldDataType.URL:
1195 annotation = Subquery(
1196 CustomFieldInstance.objects.filter(
1197 document_id=OuterRef("id"),
1198 field_id=custom_field_id,
1199 ).values("value_url")[:1],
1200 )
1201 case CustomField.FieldDataType.BOOL:
1202 annotation = Subquery(
1203 CustomFieldInstance.objects.filter(
1204 document_id=OuterRef("id"),
1205 field_id=custom_field_id,
1206 ).values("value_bool")[:1],
1207 )
1209 if not annotation:
1210 # Only happens if a new data type is added and not handled here
1211 raise ValueError("Invalid custom field data type")
1213 queryset = (
1214 queryset.annotate(
1215 # We need to annotate the queryset with the custom field value
1216 custom_field_value=annotation,
1217 # We also need to annotate the queryset with a boolean for sorting whether the field exists
1218 has_field=Exists(
1219 CustomFieldInstance.objects.filter(
1220 document_id=OuterRef("id"),
1221 field_id=custom_field_id,
1222 ),
1223 ),
1224 )
1225 .order_by(
1226 "-has_field",
1227 param.replace(
1228 self.prefix + str(custom_field_id),
1229 "custom_field_value",
1230 ),
1231 )
1232 .distinct()
1233 )
1235 return super().filter_queryset(request, queryset, view)