We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Added Last x Days not including documents added today
### Description

The "Added" filter is not including documents added today when using any of the "Last x" filter options.  I have a "Recently Added" view which does not include the most recent documents that were just processed.  If I change the filter to a specific date, for example, `After: 1/1/2023`, then the documents are included in the view.  If I change to "Last 7 days", "Last Month", "Last 3 Months" or "Last Year", then the most recent document in the view is one I scanned 3 days ago.  

### Steps to reproduce

1) Add a new document to Paperless and let it complete processing
2) Open a document view
3) Set the sort to "Added" and descending (z-a)
4) Set the Added filter to "Last 7 days"
  - The document added does not show in the view
 5) Set the Added filter to "After: 1/1/2023"
  - The document added shows in the view

### Webserver logs

```bash
No server messages logged pertaining to the issue
```


### Browser logs

```bash
No browser messages logged pertaining to the issue
```


### Paperless-ngx version

1.12.2

### Host OS

Debian 11 x86_64

### Installation method

Docker - official image

### Browser

Chrome

### Configuration changes

PAPERLESS_TIME_ZONE set to "America/Denver", Date order set to "MDY"

### Other

_No response_

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/documents/index.py`
Content:
```
1 import logging
2 import math
3 import os
4 from contextlib import contextmanager
5 
6 from dateutil.parser import isoparse
7 from django.conf import settings
8 from documents.models import Comment
9 from documents.models import Document
10 from whoosh import classify
11 from whoosh import highlight
12 from whoosh import query
13 from whoosh.fields import BOOLEAN
14 from whoosh.fields import DATETIME
15 from whoosh.fields import KEYWORD
16 from whoosh.fields import NUMERIC
17 from whoosh.fields import Schema
18 from whoosh.fields import TEXT
19 from whoosh.highlight import HtmlFormatter
20 from whoosh.index import create_in
21 from whoosh.index import exists_in
22 from whoosh.index import open_dir
23 from whoosh.qparser import MultifieldParser
24 from whoosh.qparser.dateparse import DateParserPlugin
25 from whoosh.searching import ResultsPage
26 from whoosh.searching import Searcher
27 from whoosh.writing import AsyncWriter
28 
29 logger = logging.getLogger("paperless.index")
30 
31 
32 def get_schema():
33     return Schema(
34         id=NUMERIC(stored=True, unique=True),
35         title=TEXT(sortable=True),
36         content=TEXT(),
37         asn=NUMERIC(sortable=True, signed=False),
38         correspondent=TEXT(sortable=True),
39         correspondent_id=NUMERIC(),
40         has_correspondent=BOOLEAN(),
41         tag=KEYWORD(commas=True, scorable=True, lowercase=True),
42         tag_id=KEYWORD(commas=True, scorable=True),
43         has_tag=BOOLEAN(),
44         type=TEXT(sortable=True),
45         type_id=NUMERIC(),
46         has_type=BOOLEAN(),
47         created=DATETIME(sortable=True),
48         modified=DATETIME(sortable=True),
49         added=DATETIME(sortable=True),
50         path=TEXT(sortable=True),
51         path_id=NUMERIC(),
52         has_path=BOOLEAN(),
53         comments=TEXT(),
54     )
55 
56 
57 def open_index(recreate=False):
58     try:
59         if exists_in(settings.INDEX_DIR) and not recreate:
60             return open_dir(settings.INDEX_DIR, schema=get_schema())
61     except Exception:
62         logger.exception("Error while opening the index, recreating.")
63 
64     if not os.path.isdir(settings.INDEX_DIR):
65         os.makedirs(settings.INDEX_DIR, exist_ok=True)
66     return create_in(settings.INDEX_DIR, get_schema())
67 
68 
69 @contextmanager
70 def open_index_writer(optimize=False):
71     writer = AsyncWriter(open_index())
72 
73     try:
74         yield writer
75     except Exception as e:
76         logger.exception(str(e))
77         writer.cancel()
78     finally:
79         writer.commit(optimize=optimize)
80 
81 
82 @contextmanager
83 def open_index_searcher():
84     searcher = open_index().searcher()
85 
86     try:
87         yield searcher
88     finally:
89         searcher.close()
90 
91 
92 def update_document(writer, doc):
93     tags = ",".join([t.name for t in doc.tags.all()])
94     tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
95     comments = ",".join([str(c.comment) for c in Comment.objects.filter(document=doc)])
96     writer.update_document(
97         id=doc.pk,
98         title=doc.title,
99         content=doc.content,
100         correspondent=doc.correspondent.name if doc.correspondent else None,
101         correspondent_id=doc.correspondent.id if doc.correspondent else None,
102         has_correspondent=doc.correspondent is not None,
103         tag=tags if tags else None,
104         tag_id=tags_ids if tags_ids else None,
105         has_tag=len(tags) > 0,
106         type=doc.document_type.name if doc.document_type else None,
107         type_id=doc.document_type.id if doc.document_type else None,
108         has_type=doc.document_type is not None,
109         created=doc.created,
110         added=doc.added,
111         asn=doc.archive_serial_number,
112         modified=doc.modified,
113         path=doc.storage_path.name if doc.storage_path else None,
114         path_id=doc.storage_path.id if doc.storage_path else None,
115         has_path=doc.storage_path is not None,
116         comments=comments,
117     )
118 
119 
120 def remove_document(writer, doc):
121     remove_document_by_id(writer, doc.pk)
122 
123 
124 def remove_document_by_id(writer, doc_id):
125     writer.delete_by_term("id", doc_id)
126 
127 
128 def add_or_update_document(document):
129     with open_index_writer() as writer:
130         update_document(writer, document)
131 
132 
133 def remove_document_from_index(document):
134     with open_index_writer() as writer:
135         remove_document(writer, document)
136 
137 
138 class DelayedQuery:
139     def _get_query(self):
140         raise NotImplementedError()
141 
142     def _get_query_filter(self):
143         criterias = []
144         for k, v in self.query_params.items():
145             if k == "correspondent__id":
146                 criterias.append(query.Term("correspondent_id", v))
147             elif k == "tags__id__all":
148                 for tag_id in v.split(","):
149                     criterias.append(query.Term("tag_id", tag_id))
150             elif k == "tags__id__none":
151                 for tag_id in v.split(","):
152                     criterias.append(query.Not(query.Term("tag_id", tag_id)))
153             elif k == "document_type__id":
154                 criterias.append(query.Term("type_id", v))
155             elif k == "correspondent__isnull":
156                 criterias.append(query.Term("has_correspondent", v == "false"))
157             elif k == "is_tagged":
158                 criterias.append(query.Term("has_tag", v == "true"))
159             elif k == "document_type__isnull":
160                 criterias.append(query.Term("has_type", v == "false"))
161             elif k == "created__date__lt":
162                 criterias.append(
163                     query.DateRange("created", start=None, end=isoparse(v)),
164                 )
165             elif k == "created__date__gt":
166                 criterias.append(
167                     query.DateRange("created", start=isoparse(v), end=None),
168                 )
169             elif k == "added__date__gt":
170                 criterias.append(query.DateRange("added", start=isoparse(v), end=None))
171             elif k == "added__date__lt":
172                 criterias.append(query.DateRange("added", start=None, end=isoparse(v)))
173             elif k == "storage_path__id":
174                 criterias.append(query.Term("path_id", v))
175             elif k == "storage_path__isnull":
176                 criterias.append(query.Term("has_path", v == "false"))
177 
178         if len(criterias) > 0:
179             return query.And(criterias)
180         else:
181             return None
182 
183     def _get_query_sortedby(self):
184         if "ordering" not in self.query_params:
185             return None, False
186 
187         field: str = self.query_params["ordering"]
188 
189         sort_fields_map = {
190             "created": "created",
191             "modified": "modified",
192             "added": "added",
193             "title": "title",
194             "correspondent__name": "correspondent",
195             "document_type__name": "type",
196             "archive_serial_number": "asn",
197         }
198 
199         if field.startswith("-"):
200             field = field[1:]
201             reverse = True
202         else:
203             reverse = False
204 
205         if field not in sort_fields_map:
206             return None, False
207         else:
208             return sort_fields_map[field], reverse
209 
210     def __init__(self, searcher: Searcher, query_params, page_size):
211         self.searcher = searcher
212         self.query_params = query_params
213         self.page_size = page_size
214         self.saved_results = dict()
215         self.first_score = None
216 
217     def __len__(self):
218         page = self[0:1]
219         return len(page)
220 
221     def __getitem__(self, item):
222         if item.start in self.saved_results:
223             return self.saved_results[item.start]
224 
225         q, mask = self._get_query()
226         sortedby, reverse = self._get_query_sortedby()
227 
228         page: ResultsPage = self.searcher.search_page(
229             q,
230             mask=mask,
231             filter=self._get_query_filter(),
232             pagenum=math.floor(item.start / self.page_size) + 1,
233             pagelen=self.page_size,
234             sortedby=sortedby,
235             reverse=reverse,
236         )
237         page.results.fragmenter = highlight.ContextFragmenter(surround=50)
238         page.results.formatter = HtmlFormatter(tagname="span", between=" ... ")
239 
240         if not self.first_score and len(page.results) > 0 and sortedby is None:
241             self.first_score = page.results[0].score
242 
243         page.results.top_n = list(
244             map(
245                 lambda hit: (
246                     (hit[0] / self.first_score) if self.first_score else None,
247                     hit[1],
248                 ),
249                 page.results.top_n,
250             ),
251         )
252 
253         self.saved_results[item.start] = page
254 
255         return page
256 
257 
258 class DelayedFullTextQuery(DelayedQuery):
259     def _get_query(self):
260         q_str = self.query_params["query"]
261         qp = MultifieldParser(
262             ["content", "title", "correspondent", "tag", "type", "comments"],
263             self.searcher.ixreader.schema,
264         )
265         qp.add_plugin(DateParserPlugin())
266         q = qp.parse(q_str)
267 
268         corrected = self.searcher.correct_query(q, q_str)
269         if corrected.query != q:
270             corrected.query = corrected.string
271 
272         return q, None
273 
274 
275 class DelayedMoreLikeThisQuery(DelayedQuery):
276     def _get_query(self):
277         more_like_doc_id = int(self.query_params["more_like_id"])
278         content = Document.objects.get(id=more_like_doc_id).content
279 
280         docnum = self.searcher.document_number(id=more_like_doc_id)
281         kts = self.searcher.key_terms_from_text(
282             "content",
283             content,
284             numterms=20,
285             model=classify.Bo1Model,
286             normalize=False,
287         )
288         q = query.Or(
289             [query.Term("content", word, boost=weight) for word, weight in kts],
290         )
291         mask = {docnum}
292 
293         return q, mask
294 
295 
296 def autocomplete(ix, term, limit=10):
297     with ix.reader() as reader:
298         terms = []
299         for (score, t) in reader.most_distinctive_terms(
300             "content",
301             number=limit,
302             prefix=term.lower(),
303         ):
304             terms.append(t)
305         return terms
306
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/documents/index.py b/src/documents/index.py
--- a/src/documents/index.py
+++ b/src/documents/index.py
@@ -5,6 +5,7 @@
 
 from dateutil.parser import isoparse
 from django.conf import settings
+from django.utils import timezone
 from documents.models import Comment
 from documents.models import Document
 from whoosh import classify
@@ -262,7 +263,7 @@
             ["content", "title", "correspondent", "tag", "type", "comments"],
             self.searcher.ixreader.schema,
         )
-        qp.add_plugin(DateParserPlugin())
+        qp.add_plugin(DateParserPlugin(basedate=timezone.now()))
         q = qp.parse(q_str)
 
         corrected = self.searcher.correct_query(q, q_str)

{"golden_diff": "diff --git a/src/documents/index.py b/src/documents/index.py\n--- a/src/documents/index.py\n+++ b/src/documents/index.py\n@@ -5,6 +5,7 @@\n \n from dateutil.parser import isoparse\n from django.conf import settings\n+from django.utils import timezone\n from documents.models import Comment\n from documents.models import Document\n from whoosh import classify\n@@ -262,7 +263,7 @@\n             [\"content\", \"title\", \"correspondent\", \"tag\", \"type\", \"comments\"],\n             self.searcher.ixreader.schema,\n         )\n-        qp.add_plugin(DateParserPlugin())\n+        qp.add_plugin(DateParserPlugin(basedate=timezone.now()))\n         q = qp.parse(q_str)\n \n         corrected = self.searcher.correct_query(q, q_str)\n", "issue": "Added Last x Days not including documents added today\n### Description\n\nThe \"Added\" filter is not including documents added today when using any of the \"Last x\" filter options.  I have a \"Recently Added\" view which does not include the most recent documents that were just processed.  If I change the filter to a specific date, for example, `After: 1/1/2023`, then the documents are included in the view.  If I change to \"Last 7 days\", \"Last Month\", \"Last 3 Months\" or \"Last Year\", then the most recent document in the view is one I scanned 3 days ago.  \n\n### Steps to reproduce\n\n1) Add a new document to Paperless and let it complete processing\r\n2) Open a document view\r\n3) Set the sort to \"Added\" and descending (z-a)\r\n4) Set the Added filter to \"Last 7 days\"\r\n  - The document added does not show in the view\r\n 5) Set the Added filter to \"After: 1/1/2023\"\r\n  - The document added shows in the view\n\n### Webserver logs\n\n```bash\nNo server messages logged pertaining to the issue\n```\n\n\n### Browser logs\n\n```bash\nNo browser messages logged pertaining to the issue\n```\n\n\n### Paperless-ngx version\n\n1.12.2\n\n### Host OS\n\nDebian 11 x86_64\n\n### Installation method\n\nDocker - official image\n\n### Browser\n\nChrome\n\n### Configuration changes\n\nPAPERLESS_TIME_ZONE set to \"America/Denver\", Date order set to \"MDY\"\n\n### Other\n\n_No response_\n", "before_files": [{"content": "import logging\nimport math\nimport os\nfrom contextlib import contextmanager\n\nfrom dateutil.parser import isoparse\nfrom django.conf import settings\nfrom documents.models import Comment\nfrom documents.models import Document\nfrom whoosh import classify\nfrom whoosh import highlight\nfrom whoosh import query\nfrom whoosh.fields import BOOLEAN\nfrom whoosh.fields import DATETIME\nfrom whoosh.fields import KEYWORD\nfrom whoosh.fields import NUMERIC\nfrom whoosh.fields import Schema\nfrom whoosh.fields import TEXT\nfrom whoosh.highlight import HtmlFormatter\nfrom whoosh.index import create_in\nfrom whoosh.index import exists_in\nfrom whoosh.index import open_dir\nfrom whoosh.qparser import MultifieldParser\nfrom whoosh.qparser.dateparse import DateParserPlugin\nfrom whoosh.searching import ResultsPage\nfrom whoosh.searching import Searcher\nfrom whoosh.writing import AsyncWriter\n\nlogger = logging.getLogger(\"paperless.index\")\n\n\ndef get_schema():\n    return Schema(\n        id=NUMERIC(stored=True, unique=True),\n        title=TEXT(sortable=True),\n        content=TEXT(),\n        asn=NUMERIC(sortable=True, signed=False),\n        correspondent=TEXT(sortable=True),\n        correspondent_id=NUMERIC(),\n        has_correspondent=BOOLEAN(),\n        tag=KEYWORD(commas=True, scorable=True, lowercase=True),\n        tag_id=KEYWORD(commas=True, scorable=True),\n        has_tag=BOOLEAN(),\n        type=TEXT(sortable=True),\n        type_id=NUMERIC(),\n        has_type=BOOLEAN(),\n        created=DATETIME(sortable=True),\n        modified=DATETIME(sortable=True),\n        added=DATETIME(sortable=True),\n        path=TEXT(sortable=True),\n        path_id=NUMERIC(),\n        has_path=BOOLEAN(),\n        comments=TEXT(),\n    )\n\n\ndef open_index(recreate=False):\n    try:\n        if exists_in(settings.INDEX_DIR) and not recreate:\n            return open_dir(settings.INDEX_DIR, schema=get_schema())\n    except Exception:\n        logger.exception(\"Error while opening the index, recreating.\")\n\n    if not os.path.isdir(settings.INDEX_DIR):\n        os.makedirs(settings.INDEX_DIR, exist_ok=True)\n    return create_in(settings.INDEX_DIR, get_schema())\n\n\n@contextmanager\ndef open_index_writer(optimize=False):\n    writer = AsyncWriter(open_index())\n\n    try:\n        yield writer\n    except Exception as e:\n        logger.exception(str(e))\n        writer.cancel()\n    finally:\n        writer.commit(optimize=optimize)\n\n\n@contextmanager\ndef open_index_searcher():\n    searcher = open_index().searcher()\n\n    try:\n        yield searcher\n    finally:\n        searcher.close()\n\n\ndef update_document(writer, doc):\n    tags = \",\".join([t.name for t in doc.tags.all()])\n    tags_ids = \",\".join([str(t.id) for t in doc.tags.all()])\n    comments = \",\".join([str(c.comment) for c in Comment.objects.filter(document=doc)])\n    writer.update_document(\n        id=doc.pk,\n        title=doc.title,\n        content=doc.content,\n        correspondent=doc.correspondent.name if doc.correspondent else None,\n        correspondent_id=doc.correspondent.id if doc.correspondent else None,\n        has_correspondent=doc.correspondent is not None,\n        tag=tags if tags else None,\n        tag_id=tags_ids if tags_ids else None,\n        has_tag=len(tags) > 0,\n        type=doc.document_type.name if doc.document_type else None,\n        type_id=doc.document_type.id if doc.document_type else None,\n        has_type=doc.document_type is not None,\n        created=doc.created,\n        added=doc.added,\n        asn=doc.archive_serial_number,\n        modified=doc.modified,\n        path=doc.storage_path.name if doc.storage_path else None,\n        path_id=doc.storage_path.id if doc.storage_path else None,\n        has_path=doc.storage_path is not None,\n        comments=comments,\n    )\n\n\ndef remove_document(writer, doc):\n    remove_document_by_id(writer, doc.pk)\n\n\ndef remove_document_by_id(writer, doc_id):\n    writer.delete_by_term(\"id\", doc_id)\n\n\ndef add_or_update_document(document):\n    with open_index_writer() as writer:\n        update_document(writer, document)\n\n\ndef remove_document_from_index(document):\n    with open_index_writer() as writer:\n        remove_document(writer, document)\n\n\nclass DelayedQuery:\n    def _get_query(self):\n        raise NotImplementedError()\n\n    def _get_query_filter(self):\n        criterias = []\n        for k, v in self.query_params.items():\n            if k == \"correspondent__id\":\n                criterias.append(query.Term(\"correspondent_id\", v))\n            elif k == \"tags__id__all\":\n                for tag_id in v.split(\",\"):\n                    criterias.append(query.Term(\"tag_id\", tag_id))\n            elif k == \"tags__id__none\":\n                for tag_id in v.split(\",\"):\n                    criterias.append(query.Not(query.Term(\"tag_id\", tag_id)))\n            elif k == \"document_type__id\":\n                criterias.append(query.Term(\"type_id\", v))\n            elif k == \"correspondent__isnull\":\n                criterias.append(query.Term(\"has_correspondent\", v == \"false\"))\n            elif k == \"is_tagged\":\n                criterias.append(query.Term(\"has_tag\", v == \"true\"))\n            elif k == \"document_type__isnull\":\n                criterias.append(query.Term(\"has_type\", v == \"false\"))\n            elif k == \"created__date__lt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=None, end=isoparse(v)),\n                )\n            elif k == \"created__date__gt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=isoparse(v), end=None),\n                )\n            elif k == \"added__date__gt\":\n                criterias.append(query.DateRange(\"added\", start=isoparse(v), end=None))\n            elif k == \"added__date__lt\":\n                criterias.append(query.DateRange(\"added\", start=None, end=isoparse(v)))\n            elif k == \"storage_path__id\":\n                criterias.append(query.Term(\"path_id\", v))\n            elif k == \"storage_path__isnull\":\n                criterias.append(query.Term(\"has_path\", v == \"false\"))\n\n        if len(criterias) > 0:\n            return query.And(criterias)\n        else:\n            return None\n\n    def _get_query_sortedby(self):\n        if \"ordering\" not in self.query_params:\n            return None, False\n\n        field: str = self.query_params[\"ordering\"]\n\n        sort_fields_map = {\n            \"created\": \"created\",\n            \"modified\": \"modified\",\n            \"added\": \"added\",\n            \"title\": \"title\",\n            \"correspondent__name\": \"correspondent\",\n            \"document_type__name\": \"type\",\n            \"archive_serial_number\": \"asn\",\n        }\n\n        if field.startswith(\"-\"):\n            field = field[1:]\n            reverse = True\n        else:\n            reverse = False\n\n        if field not in sort_fields_map:\n            return None, False\n        else:\n            return sort_fields_map[field], reverse\n\n    def __init__(self, searcher: Searcher, query_params, page_size):\n        self.searcher = searcher\n        self.query_params = query_params\n        self.page_size = page_size\n        self.saved_results = dict()\n        self.first_score = None\n\n    def __len__(self):\n        page = self[0:1]\n        return len(page)\n\n    def __getitem__(self, item):\n        if item.start in self.saved_results:\n            return self.saved_results[item.start]\n\n        q, mask = self._get_query()\n        sortedby, reverse = self._get_query_sortedby()\n\n        page: ResultsPage = self.searcher.search_page(\n            q,\n            mask=mask,\n            filter=self._get_query_filter(),\n            pagenum=math.floor(item.start / self.page_size) + 1,\n            pagelen=self.page_size,\n            sortedby=sortedby,\n            reverse=reverse,\n        )\n        page.results.fragmenter = highlight.ContextFragmenter(surround=50)\n        page.results.formatter = HtmlFormatter(tagname=\"span\", between=\" ... \")\n\n        if not self.first_score and len(page.results) > 0 and sortedby is None:\n            self.first_score = page.results[0].score\n\n        page.results.top_n = list(\n            map(\n                lambda hit: (\n                    (hit[0] / self.first_score) if self.first_score else None,\n                    hit[1],\n                ),\n                page.results.top_n,\n            ),\n        )\n\n        self.saved_results[item.start] = page\n\n        return page\n\n\nclass DelayedFullTextQuery(DelayedQuery):\n    def _get_query(self):\n        q_str = self.query_params[\"query\"]\n        qp = MultifieldParser(\n            [\"content\", \"title\", \"correspondent\", \"tag\", \"type\", \"comments\"],\n            self.searcher.ixreader.schema,\n        )\n        qp.add_plugin(DateParserPlugin())\n        q = qp.parse(q_str)\n\n        corrected = self.searcher.correct_query(q, q_str)\n        if corrected.query != q:\n            corrected.query = corrected.string\n\n        return q, None\n\n\nclass DelayedMoreLikeThisQuery(DelayedQuery):\n    def _get_query(self):\n        more_like_doc_id = int(self.query_params[\"more_like_id\"])\n        content = Document.objects.get(id=more_like_doc_id).content\n\n        docnum = self.searcher.document_number(id=more_like_doc_id)\n        kts = self.searcher.key_terms_from_text(\n            \"content\",\n            content,\n            numterms=20,\n            model=classify.Bo1Model,\n            normalize=False,\n        )\n        q = query.Or(\n            [query.Term(\"content\", word, boost=weight) for word, weight in kts],\n        )\n        mask = {docnum}\n\n        return q, mask\n\n\ndef autocomplete(ix, term, limit=10):\n    with ix.reader() as reader:\n        terms = []\n        for (score, t) in reader.most_distinctive_terms(\n            \"content\",\n            number=limit,\n            prefix=term.lower(),\n        ):\n            terms.append(t)\n        return terms\n", "path": "src/documents/index.py"}], "after_files": [{"content": "import logging\nimport math\nimport os\nfrom contextlib import contextmanager\n\nfrom dateutil.parser import isoparse\nfrom django.conf import settings\nfrom django.utils import timezone\nfrom documents.models import Comment\nfrom documents.models import Document\nfrom whoosh import classify\nfrom whoosh import highlight\nfrom whoosh import query\nfrom whoosh.fields import BOOLEAN\nfrom whoosh.fields import DATETIME\nfrom whoosh.fields import KEYWORD\nfrom whoosh.fields import NUMERIC\nfrom whoosh.fields import Schema\nfrom whoosh.fields import TEXT\nfrom whoosh.highlight import HtmlFormatter\nfrom whoosh.index import create_in\nfrom whoosh.index import exists_in\nfrom whoosh.index import open_dir\nfrom whoosh.qparser import MultifieldParser\nfrom whoosh.qparser.dateparse import DateParserPlugin\nfrom whoosh.searching import ResultsPage\nfrom whoosh.searching import Searcher\nfrom whoosh.writing import AsyncWriter\n\nlogger = logging.getLogger(\"paperless.index\")\n\n\ndef get_schema():\n    return Schema(\n        id=NUMERIC(stored=True, unique=True),\n        title=TEXT(sortable=True),\n        content=TEXT(),\n        asn=NUMERIC(sortable=True, signed=False),\n        correspondent=TEXT(sortable=True),\n        correspondent_id=NUMERIC(),\n        has_correspondent=BOOLEAN(),\n        tag=KEYWORD(commas=True, scorable=True, lowercase=True),\n        tag_id=KEYWORD(commas=True, scorable=True),\n        has_tag=BOOLEAN(),\n        type=TEXT(sortable=True),\n        type_id=NUMERIC(),\n        has_type=BOOLEAN(),\n        created=DATETIME(sortable=True),\n        modified=DATETIME(sortable=True),\n        added=DATETIME(sortable=True),\n        path=TEXT(sortable=True),\n        path_id=NUMERIC(),\n        has_path=BOOLEAN(),\n        comments=TEXT(),\n    )\n\n\ndef open_index(recreate=False):\n    try:\n        if exists_in(settings.INDEX_DIR) and not recreate:\n            return open_dir(settings.INDEX_DIR, schema=get_schema())\n    except Exception:\n        logger.exception(\"Error while opening the index, recreating.\")\n\n    if not os.path.isdir(settings.INDEX_DIR):\n        os.makedirs(settings.INDEX_DIR, exist_ok=True)\n    return create_in(settings.INDEX_DIR, get_schema())\n\n\n@contextmanager\ndef open_index_writer(optimize=False):\n    writer = AsyncWriter(open_index())\n\n    try:\n        yield writer\n    except Exception as e:\n        logger.exception(str(e))\n        writer.cancel()\n    finally:\n        writer.commit(optimize=optimize)\n\n\n@contextmanager\ndef open_index_searcher():\n    searcher = open_index().searcher()\n\n    try:\n        yield searcher\n    finally:\n        searcher.close()\n\n\ndef update_document(writer, doc):\n    tags = \",\".join([t.name for t in doc.tags.all()])\n    tags_ids = \",\".join([str(t.id) for t in doc.tags.all()])\n    comments = \",\".join([str(c.comment) for c in Comment.objects.filter(document=doc)])\n    writer.update_document(\n        id=doc.pk,\n        title=doc.title,\n        content=doc.content,\n        correspondent=doc.correspondent.name if doc.correspondent else None,\n        correspondent_id=doc.correspondent.id if doc.correspondent else None,\n        has_correspondent=doc.correspondent is not None,\n        tag=tags if tags else None,\n        tag_id=tags_ids if tags_ids else None,\n        has_tag=len(tags) > 0,\n        type=doc.document_type.name if doc.document_type else None,\n        type_id=doc.document_type.id if doc.document_type else None,\n        has_type=doc.document_type is not None,\n        created=doc.created,\n        added=doc.added,\n        asn=doc.archive_serial_number,\n        modified=doc.modified,\n        path=doc.storage_path.name if doc.storage_path else None,\n        path_id=doc.storage_path.id if doc.storage_path else None,\n        has_path=doc.storage_path is not None,\n        comments=comments,\n    )\n\n\ndef remove_document(writer, doc):\n    remove_document_by_id(writer, doc.pk)\n\n\ndef remove_document_by_id(writer, doc_id):\n    writer.delete_by_term(\"id\", doc_id)\n\n\ndef add_or_update_document(document):\n    with open_index_writer() as writer:\n        update_document(writer, document)\n\n\ndef remove_document_from_index(document):\n    with open_index_writer() as writer:\n        remove_document(writer, document)\n\n\nclass DelayedQuery:\n    def _get_query(self):\n        raise NotImplementedError()\n\n    def _get_query_filter(self):\n        criterias = []\n        for k, v in self.query_params.items():\n            if k == \"correspondent__id\":\n                criterias.append(query.Term(\"correspondent_id\", v))\n            elif k == \"tags__id__all\":\n                for tag_id in v.split(\",\"):\n                    criterias.append(query.Term(\"tag_id\", tag_id))\n            elif k == \"tags__id__none\":\n                for tag_id in v.split(\",\"):\n                    criterias.append(query.Not(query.Term(\"tag_id\", tag_id)))\n            elif k == \"document_type__id\":\n                criterias.append(query.Term(\"type_id\", v))\n            elif k == \"correspondent__isnull\":\n                criterias.append(query.Term(\"has_correspondent\", v == \"false\"))\n            elif k == \"is_tagged\":\n                criterias.append(query.Term(\"has_tag\", v == \"true\"))\n            elif k == \"document_type__isnull\":\n                criterias.append(query.Term(\"has_type\", v == \"false\"))\n            elif k == \"created__date__lt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=None, end=isoparse(v)),\n                )\n            elif k == \"created__date__gt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=isoparse(v), end=None),\n                )\n            elif k == \"added__date__gt\":\n                criterias.append(query.DateRange(\"added\", start=isoparse(v), end=None))\n            elif k == \"added__date__lt\":\n                criterias.append(query.DateRange(\"added\", start=None, end=isoparse(v)))\n            elif k == \"storage_path__id\":\n                criterias.append(query.Term(\"path_id\", v))\n            elif k == \"storage_path__isnull\":\n                criterias.append(query.Term(\"has_path\", v == \"false\"))\n\n        if len(criterias) > 0:\n            return query.And(criterias)\n        else:\n            return None\n\n    def _get_query_sortedby(self):\n        if \"ordering\" not in self.query_params:\n            return None, False\n\n        field: str = self.query_params[\"ordering\"]\n\n        sort_fields_map = {\n            \"created\": \"created\",\n            \"modified\": \"modified\",\n            \"added\": \"added\",\n            \"title\": \"title\",\n            \"correspondent__name\": \"correspondent\",\n            \"document_type__name\": \"type\",\n            \"archive_serial_number\": \"asn\",\n        }\n\n        if field.startswith(\"-\"):\n            field = field[1:]\n            reverse = True\n        else:\n            reverse = False\n\n        if field not in sort_fields_map:\n            return None, False\n        else:\n            return sort_fields_map[field], reverse\n\n    def __init__(self, searcher: Searcher, query_params, page_size):\n        self.searcher = searcher\n        self.query_params = query_params\n        self.page_size = page_size\n        self.saved_results = dict()\n        self.first_score = None\n\n    def __len__(self):\n        page = self[0:1]\n        return len(page)\n\n    def __getitem__(self, item):\n        if item.start in self.saved_results:\n            return self.saved_results[item.start]\n\n        q, mask = self._get_query()\n        sortedby, reverse = self._get_query_sortedby()\n\n        page: ResultsPage = self.searcher.search_page(\n            q,\n            mask=mask,\n            filter=self._get_query_filter(),\n            pagenum=math.floor(item.start / self.page_size) + 1,\n            pagelen=self.page_size,\n            sortedby=sortedby,\n            reverse=reverse,\n        )\n        page.results.fragmenter = highlight.ContextFragmenter(surround=50)\n        page.results.formatter = HtmlFormatter(tagname=\"span\", between=\" ... \")\n\n        if not self.first_score and len(page.results) > 0 and sortedby is None:\n            self.first_score = page.results[0].score\n\n        page.results.top_n = list(\n            map(\n                lambda hit: (\n                    (hit[0] / self.first_score) if self.first_score else None,\n                    hit[1],\n                ),\n                page.results.top_n,\n            ),\n        )\n\n        self.saved_results[item.start] = page\n\n        return page\n\n\nclass DelayedFullTextQuery(DelayedQuery):\n    def _get_query(self):\n        q_str = self.query_params[\"query\"]\n        qp = MultifieldParser(\n            [\"content\", \"title\", \"correspondent\", \"tag\", \"type\", \"comments\"],\n            self.searcher.ixreader.schema,\n        )\n        qp.add_plugin(DateParserPlugin(basedate=timezone.now()))\n        q = qp.parse(q_str)\n\n        corrected = self.searcher.correct_query(q, q_str)\n        if corrected.query != q:\n            corrected.query = corrected.string\n\n        return q, None\n\n\nclass DelayedMoreLikeThisQuery(DelayedQuery):\n    def _get_query(self):\n        more_like_doc_id = int(self.query_params[\"more_like_id\"])\n        content = Document.objects.get(id=more_like_doc_id).content\n\n        docnum = self.searcher.document_number(id=more_like_doc_id)\n        kts = self.searcher.key_terms_from_text(\n            \"content\",\n            content,\n            numterms=20,\n            model=classify.Bo1Model,\n            normalize=False,\n        )\n        q = query.Or(\n            [query.Term(\"content\", word, boost=weight) for word, weight in kts],\n        )\n        mask = {docnum}\n\n        return q, mask\n\n\ndef autocomplete(ix, term, limit=10):\n    with ix.reader() as reader:\n        terms = []\n        for (score, t) in reader.most_distinctive_terms(\n            \"content\",\n            number=limit,\n            prefix=term.lower(),\n        ):\n            terms.append(t)\n        return terms\n", "path": "src/documents/index.py"}]}