We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[BUG] Filtering is inconsistent: tag exclusion does not apply with search term
### Description

Filtering documents is not working as expected in some cases.

When filtering for tags, it is possible to include/exclude or exactly match tags. That functionality is lost when using a search term at the same time. I assume that this is not intended behaviour.

### Steps to reproduce

1. Go to `Documents`
2. Enter a search term for filtering documents by that search term. This works fine.
3. Exclude a tag from the search results. The tag is not excluded, when a search term was entered.

### Webserver logs

```bash
Does not apply
```


### Browser logs

_No response_

### Paperless-ngx version

1.10.2

### Host OS

Ubuntu 22.04

### Installation method

Docker - official image

### Browser

Firefox

### Configuration changes

_No response_

### Other

_No response_

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `src/documents/index.py`
Content:
```
1 import logging
2 import math
3 import os
4 from contextlib import contextmanager
5 
6 from dateutil.parser import isoparse
7 from django.conf import settings
8 from documents.models import Document
9 from whoosh import classify
10 from whoosh import highlight
11 from whoosh import query
12 from whoosh.fields import BOOLEAN
13 from whoosh.fields import DATETIME
14 from whoosh.fields import KEYWORD
15 from whoosh.fields import NUMERIC
16 from whoosh.fields import Schema
17 from whoosh.fields import TEXT
18 from whoosh.highlight import HtmlFormatter
19 from whoosh.index import create_in
20 from whoosh.index import exists_in
21 from whoosh.index import open_dir
22 from whoosh.qparser import MultifieldParser
23 from whoosh.qparser.dateparse import DateParserPlugin
24 from whoosh.searching import ResultsPage
25 from whoosh.searching import Searcher
26 from whoosh.writing import AsyncWriter
27 
28 logger = logging.getLogger("paperless.index")
29 
30 
31 def get_schema():
32     return Schema(
33         id=NUMERIC(stored=True, unique=True),
34         title=TEXT(sortable=True),
35         content=TEXT(),
36         asn=NUMERIC(sortable=True),
37         correspondent=TEXT(sortable=True),
38         correspondent_id=NUMERIC(),
39         has_correspondent=BOOLEAN(),
40         tag=KEYWORD(commas=True, scorable=True, lowercase=True),
41         tag_id=KEYWORD(commas=True, scorable=True),
42         has_tag=BOOLEAN(),
43         type=TEXT(sortable=True),
44         type_id=NUMERIC(),
45         has_type=BOOLEAN(),
46         created=DATETIME(sortable=True),
47         modified=DATETIME(sortable=True),
48         added=DATETIME(sortable=True),
49         path=TEXT(sortable=True),
50         path_id=NUMERIC(),
51         has_path=BOOLEAN(),
52     )
53 
54 
55 def open_index(recreate=False):
56     try:
57         if exists_in(settings.INDEX_DIR) and not recreate:
58             return open_dir(settings.INDEX_DIR, schema=get_schema())
59     except Exception:
60         logger.exception("Error while opening the index, recreating.")
61 
62     if not os.path.isdir(settings.INDEX_DIR):
63         os.makedirs(settings.INDEX_DIR, exist_ok=True)
64     return create_in(settings.INDEX_DIR, get_schema())
65 
66 
67 @contextmanager
68 def open_index_writer(optimize=False):
69     writer = AsyncWriter(open_index())
70 
71     try:
72         yield writer
73     except Exception as e:
74         logger.exception(str(e))
75         writer.cancel()
76     finally:
77         writer.commit(optimize=optimize)
78 
79 
80 @contextmanager
81 def open_index_searcher():
82     searcher = open_index().searcher()
83 
84     try:
85         yield searcher
86     finally:
87         searcher.close()
88 
89 
90 def update_document(writer, doc):
91     tags = ",".join([t.name for t in doc.tags.all()])
92     tags_ids = ",".join([str(t.id) for t in doc.tags.all()])
93     writer.update_document(
94         id=doc.pk,
95         title=doc.title,
96         content=doc.content,
97         correspondent=doc.correspondent.name if doc.correspondent else None,
98         correspondent_id=doc.correspondent.id if doc.correspondent else None,
99         has_correspondent=doc.correspondent is not None,
100         tag=tags if tags else None,
101         tag_id=tags_ids if tags_ids else None,
102         has_tag=len(tags) > 0,
103         type=doc.document_type.name if doc.document_type else None,
104         type_id=doc.document_type.id if doc.document_type else None,
105         has_type=doc.document_type is not None,
106         created=doc.created,
107         added=doc.added,
108         asn=doc.archive_serial_number,
109         modified=doc.modified,
110         path=doc.storage_path.name if doc.storage_path else None,
111         path_id=doc.storage_path.id if doc.storage_path else None,
112         has_path=doc.storage_path is not None,
113     )
114 
115 
116 def remove_document(writer, doc):
117     remove_document_by_id(writer, doc.pk)
118 
119 
120 def remove_document_by_id(writer, doc_id):
121     writer.delete_by_term("id", doc_id)
122 
123 
124 def add_or_update_document(document):
125     with open_index_writer() as writer:
126         update_document(writer, document)
127 
128 
129 def remove_document_from_index(document):
130     with open_index_writer() as writer:
131         remove_document(writer, document)
132 
133 
134 class DelayedQuery:
135     def _get_query(self):
136         raise NotImplementedError()
137 
138     def _get_query_filter(self):
139         criterias = []
140         for k, v in self.query_params.items():
141             if k == "correspondent__id":
142                 criterias.append(query.Term("correspondent_id", v))
143             elif k == "tags__id__all":
144                 for tag_id in v.split(","):
145                     criterias.append(query.Term("tag_id", tag_id))
146             elif k == "document_type__id":
147                 criterias.append(query.Term("type_id", v))
148             elif k == "correspondent__isnull":
149                 criterias.append(query.Term("has_correspondent", v == "false"))
150             elif k == "is_tagged":
151                 criterias.append(query.Term("has_tag", v == "true"))
152             elif k == "document_type__isnull":
153                 criterias.append(query.Term("has_type", v == "false"))
154             elif k == "created__date__lt":
155                 criterias.append(
156                     query.DateRange("created", start=None, end=isoparse(v)),
157                 )
158             elif k == "created__date__gt":
159                 criterias.append(
160                     query.DateRange("created", start=isoparse(v), end=None),
161                 )
162             elif k == "added__date__gt":
163                 criterias.append(query.DateRange("added", start=isoparse(v), end=None))
164             elif k == "added__date__lt":
165                 criterias.append(query.DateRange("added", start=None, end=isoparse(v)))
166             elif k == "storage_path__id":
167                 criterias.append(query.Term("path_id", v))
168             elif k == "storage_path__isnull":
169                 criterias.append(query.Term("has_path", v == "false"))
170 
171         if len(criterias) > 0:
172             return query.And(criterias)
173         else:
174             return None
175 
176     def _get_query_sortedby(self):
177         if "ordering" not in self.query_params:
178             return None, False
179 
180         field: str = self.query_params["ordering"]
181 
182         sort_fields_map = {
183             "created": "created",
184             "modified": "modified",
185             "added": "added",
186             "title": "title",
187             "correspondent__name": "correspondent",
188             "document_type__name": "type",
189             "archive_serial_number": "asn",
190         }
191 
192         if field.startswith("-"):
193             field = field[1:]
194             reverse = True
195         else:
196             reverse = False
197 
198         if field not in sort_fields_map:
199             return None, False
200         else:
201             return sort_fields_map[field], reverse
202 
203     def __init__(self, searcher: Searcher, query_params, page_size):
204         self.searcher = searcher
205         self.query_params = query_params
206         self.page_size = page_size
207         self.saved_results = dict()
208         self.first_score = None
209 
210     def __len__(self):
211         page = self[0:1]
212         return len(page)
213 
214     def __getitem__(self, item):
215         if item.start in self.saved_results:
216             return self.saved_results[item.start]
217 
218         q, mask = self._get_query()
219         sortedby, reverse = self._get_query_sortedby()
220 
221         page: ResultsPage = self.searcher.search_page(
222             q,
223             mask=mask,
224             filter=self._get_query_filter(),
225             pagenum=math.floor(item.start / self.page_size) + 1,
226             pagelen=self.page_size,
227             sortedby=sortedby,
228             reverse=reverse,
229         )
230         page.results.fragmenter = highlight.ContextFragmenter(surround=50)
231         page.results.formatter = HtmlFormatter(tagname="span", between=" ... ")
232 
233         if not self.first_score and len(page.results) > 0 and sortedby is None:
234             self.first_score = page.results[0].score
235 
236         page.results.top_n = list(
237             map(
238                 lambda hit: (
239                     (hit[0] / self.first_score) if self.first_score else None,
240                     hit[1],
241                 ),
242                 page.results.top_n,
243             ),
244         )
245 
246         self.saved_results[item.start] = page
247 
248         return page
249 
250 
251 class DelayedFullTextQuery(DelayedQuery):
252     def _get_query(self):
253         q_str = self.query_params["query"]
254         qp = MultifieldParser(
255             ["content", "title", "correspondent", "tag", "type"],
256             self.searcher.ixreader.schema,
257         )
258         qp.add_plugin(DateParserPlugin())
259         q = qp.parse(q_str)
260 
261         corrected = self.searcher.correct_query(q, q_str)
262         if corrected.query != q:
263             corrected.query = corrected.string
264 
265         return q, None
266 
267 
268 class DelayedMoreLikeThisQuery(DelayedQuery):
269     def _get_query(self):
270         more_like_doc_id = int(self.query_params["more_like_id"])
271         content = Document.objects.get(id=more_like_doc_id).content
272 
273         docnum = self.searcher.document_number(id=more_like_doc_id)
274         kts = self.searcher.key_terms_from_text(
275             "content",
276             content,
277             numterms=20,
278             model=classify.Bo1Model,
279             normalize=False,
280         )
281         q = query.Or(
282             [query.Term("content", word, boost=weight) for word, weight in kts],
283         )
284         mask = {docnum}
285 
286         return q, mask
287 
288 
289 def autocomplete(ix, term, limit=10):
290     with ix.reader() as reader:
291         terms = []
292         for (score, t) in reader.most_distinctive_terms(
293             "content",
294             number=limit,
295             prefix=term.lower(),
296         ):
297             terms.append(t)
298         return terms
299
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/src/documents/index.py b/src/documents/index.py
--- a/src/documents/index.py
+++ b/src/documents/index.py
@@ -143,6 +143,9 @@
             elif k == "tags__id__all":
                 for tag_id in v.split(","):
                     criterias.append(query.Term("tag_id", tag_id))
+            elif k == "tags__id__none":
+                for tag_id in v.split(","):
+                    criterias.append(query.Not(query.Term("tag_id", tag_id)))
             elif k == "document_type__id":
                 criterias.append(query.Term("type_id", v))
             elif k == "correspondent__isnull":

{"golden_diff": "diff --git a/src/documents/index.py b/src/documents/index.py\n--- a/src/documents/index.py\n+++ b/src/documents/index.py\n@@ -143,6 +143,9 @@\n             elif k == \"tags__id__all\":\n                 for tag_id in v.split(\",\"):\n                     criterias.append(query.Term(\"tag_id\", tag_id))\n+            elif k == \"tags__id__none\":\n+                for tag_id in v.split(\",\"):\n+                    criterias.append(query.Not(query.Term(\"tag_id\", tag_id)))\n             elif k == \"document_type__id\":\n                 criterias.append(query.Term(\"type_id\", v))\n             elif k == \"correspondent__isnull\":\n", "issue": "[BUG] Filtering is inconsistent: tag exclusion does not apply with search term\n### Description\n\nFiltering documents is not working as expected in some cases.\r\n\r\nWhen filtering for tags, it is possible to include/exclude or exactly match tags. That functionality is lost when using a search term at the same time. I assume that this is not intended behaviour.\n\n### Steps to reproduce\n\n1. Go to `Documents`\r\n2. Enter a search term for filtering documents by that search term. This works fine.\r\n3. Exclude a tag from the search results. The tag is not excluded, when a search term was entered.\n\n### Webserver logs\n\n```bash\nDoes not apply\n```\n\n\n### Browser logs\n\n_No response_\n\n### Paperless-ngx version\n\n1.10.2\n\n### Host OS\n\nUbuntu 22.04\n\n### Installation method\n\nDocker - official image\n\n### Browser\n\nFirefox\n\n### Configuration changes\n\n_No response_\n\n### Other\n\n_No response_\n", "before_files": [{"content": "import logging\nimport math\nimport os\nfrom contextlib import contextmanager\n\nfrom dateutil.parser import isoparse\nfrom django.conf import settings\nfrom documents.models import Document\nfrom whoosh import classify\nfrom whoosh import highlight\nfrom whoosh import query\nfrom whoosh.fields import BOOLEAN\nfrom whoosh.fields import DATETIME\nfrom whoosh.fields import KEYWORD\nfrom whoosh.fields import NUMERIC\nfrom whoosh.fields import Schema\nfrom whoosh.fields import TEXT\nfrom whoosh.highlight import HtmlFormatter\nfrom whoosh.index import create_in\nfrom whoosh.index import exists_in\nfrom whoosh.index import open_dir\nfrom whoosh.qparser import MultifieldParser\nfrom whoosh.qparser.dateparse import DateParserPlugin\nfrom whoosh.searching import ResultsPage\nfrom whoosh.searching import Searcher\nfrom whoosh.writing import AsyncWriter\n\nlogger = logging.getLogger(\"paperless.index\")\n\n\ndef get_schema():\n    return Schema(\n        id=NUMERIC(stored=True, unique=True),\n        title=TEXT(sortable=True),\n        content=TEXT(),\n        asn=NUMERIC(sortable=True),\n        correspondent=TEXT(sortable=True),\n        correspondent_id=NUMERIC(),\n        has_correspondent=BOOLEAN(),\n        tag=KEYWORD(commas=True, scorable=True, lowercase=True),\n        tag_id=KEYWORD(commas=True, scorable=True),\n        has_tag=BOOLEAN(),\n        type=TEXT(sortable=True),\n        type_id=NUMERIC(),\n        has_type=BOOLEAN(),\n        created=DATETIME(sortable=True),\n        modified=DATETIME(sortable=True),\n        added=DATETIME(sortable=True),\n        path=TEXT(sortable=True),\n        path_id=NUMERIC(),\n        has_path=BOOLEAN(),\n    )\n\n\ndef open_index(recreate=False):\n    try:\n        if exists_in(settings.INDEX_DIR) and not recreate:\n            return open_dir(settings.INDEX_DIR, schema=get_schema())\n    except Exception:\n        logger.exception(\"Error while opening the index, recreating.\")\n\n    if not os.path.isdir(settings.INDEX_DIR):\n        os.makedirs(settings.INDEX_DIR, exist_ok=True)\n    return create_in(settings.INDEX_DIR, get_schema())\n\n\n@contextmanager\ndef open_index_writer(optimize=False):\n    writer = AsyncWriter(open_index())\n\n    try:\n        yield writer\n    except Exception as e:\n        logger.exception(str(e))\n        writer.cancel()\n    finally:\n        writer.commit(optimize=optimize)\n\n\n@contextmanager\ndef open_index_searcher():\n    searcher = open_index().searcher()\n\n    try:\n        yield searcher\n    finally:\n        searcher.close()\n\n\ndef update_document(writer, doc):\n    tags = \",\".join([t.name for t in doc.tags.all()])\n    tags_ids = \",\".join([str(t.id) for t in doc.tags.all()])\n    writer.update_document(\n        id=doc.pk,\n        title=doc.title,\n        content=doc.content,\n        correspondent=doc.correspondent.name if doc.correspondent else None,\n        correspondent_id=doc.correspondent.id if doc.correspondent else None,\n        has_correspondent=doc.correspondent is not None,\n        tag=tags if tags else None,\n        tag_id=tags_ids if tags_ids else None,\n        has_tag=len(tags) > 0,\n        type=doc.document_type.name if doc.document_type else None,\n        type_id=doc.document_type.id if doc.document_type else None,\n        has_type=doc.document_type is not None,\n        created=doc.created,\n        added=doc.added,\n        asn=doc.archive_serial_number,\n        modified=doc.modified,\n        path=doc.storage_path.name if doc.storage_path else None,\n        path_id=doc.storage_path.id if doc.storage_path else None,\n        has_path=doc.storage_path is not None,\n    )\n\n\ndef remove_document(writer, doc):\n    remove_document_by_id(writer, doc.pk)\n\n\ndef remove_document_by_id(writer, doc_id):\n    writer.delete_by_term(\"id\", doc_id)\n\n\ndef add_or_update_document(document):\n    with open_index_writer() as writer:\n        update_document(writer, document)\n\n\ndef remove_document_from_index(document):\n    with open_index_writer() as writer:\n        remove_document(writer, document)\n\n\nclass DelayedQuery:\n    def _get_query(self):\n        raise NotImplementedError()\n\n    def _get_query_filter(self):\n        criterias = []\n        for k, v in self.query_params.items():\n            if k == \"correspondent__id\":\n                criterias.append(query.Term(\"correspondent_id\", v))\n            elif k == \"tags__id__all\":\n                for tag_id in v.split(\",\"):\n                    criterias.append(query.Term(\"tag_id\", tag_id))\n            elif k == \"document_type__id\":\n                criterias.append(query.Term(\"type_id\", v))\n            elif k == \"correspondent__isnull\":\n                criterias.append(query.Term(\"has_correspondent\", v == \"false\"))\n            elif k == \"is_tagged\":\n                criterias.append(query.Term(\"has_tag\", v == \"true\"))\n            elif k == \"document_type__isnull\":\n                criterias.append(query.Term(\"has_type\", v == \"false\"))\n            elif k == \"created__date__lt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=None, end=isoparse(v)),\n                )\n            elif k == \"created__date__gt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=isoparse(v), end=None),\n                )\n            elif k == \"added__date__gt\":\n                criterias.append(query.DateRange(\"added\", start=isoparse(v), end=None))\n            elif k == \"added__date__lt\":\n                criterias.append(query.DateRange(\"added\", start=None, end=isoparse(v)))\n            elif k == \"storage_path__id\":\n                criterias.append(query.Term(\"path_id\", v))\n            elif k == \"storage_path__isnull\":\n                criterias.append(query.Term(\"has_path\", v == \"false\"))\n\n        if len(criterias) > 0:\n            return query.And(criterias)\n        else:\n            return None\n\n    def _get_query_sortedby(self):\n        if \"ordering\" not in self.query_params:\n            return None, False\n\n        field: str = self.query_params[\"ordering\"]\n\n        sort_fields_map = {\n            \"created\": \"created\",\n            \"modified\": \"modified\",\n            \"added\": \"added\",\n            \"title\": \"title\",\n            \"correspondent__name\": \"correspondent\",\n            \"document_type__name\": \"type\",\n            \"archive_serial_number\": \"asn\",\n        }\n\n        if field.startswith(\"-\"):\n            field = field[1:]\n            reverse = True\n        else:\n            reverse = False\n\n        if field not in sort_fields_map:\n            return None, False\n        else:\n            return sort_fields_map[field], reverse\n\n    def __init__(self, searcher: Searcher, query_params, page_size):\n        self.searcher = searcher\n        self.query_params = query_params\n        self.page_size = page_size\n        self.saved_results = dict()\n        self.first_score = None\n\n    def __len__(self):\n        page = self[0:1]\n        return len(page)\n\n    def __getitem__(self, item):\n        if item.start in self.saved_results:\n            return self.saved_results[item.start]\n\n        q, mask = self._get_query()\n        sortedby, reverse = self._get_query_sortedby()\n\n        page: ResultsPage = self.searcher.search_page(\n            q,\n            mask=mask,\n            filter=self._get_query_filter(),\n            pagenum=math.floor(item.start / self.page_size) + 1,\n            pagelen=self.page_size,\n            sortedby=sortedby,\n            reverse=reverse,\n        )\n        page.results.fragmenter = highlight.ContextFragmenter(surround=50)\n        page.results.formatter = HtmlFormatter(tagname=\"span\", between=\" ... \")\n\n        if not self.first_score and len(page.results) > 0 and sortedby is None:\n            self.first_score = page.results[0].score\n\n        page.results.top_n = list(\n            map(\n                lambda hit: (\n                    (hit[0] / self.first_score) if self.first_score else None,\n                    hit[1],\n                ),\n                page.results.top_n,\n            ),\n        )\n\n        self.saved_results[item.start] = page\n\n        return page\n\n\nclass DelayedFullTextQuery(DelayedQuery):\n    def _get_query(self):\n        q_str = self.query_params[\"query\"]\n        qp = MultifieldParser(\n            [\"content\", \"title\", \"correspondent\", \"tag\", \"type\"],\n            self.searcher.ixreader.schema,\n        )\n        qp.add_plugin(DateParserPlugin())\n        q = qp.parse(q_str)\n\n        corrected = self.searcher.correct_query(q, q_str)\n        if corrected.query != q:\n            corrected.query = corrected.string\n\n        return q, None\n\n\nclass DelayedMoreLikeThisQuery(DelayedQuery):\n    def _get_query(self):\n        more_like_doc_id = int(self.query_params[\"more_like_id\"])\n        content = Document.objects.get(id=more_like_doc_id).content\n\n        docnum = self.searcher.document_number(id=more_like_doc_id)\n        kts = self.searcher.key_terms_from_text(\n            \"content\",\n            content,\n            numterms=20,\n            model=classify.Bo1Model,\n            normalize=False,\n        )\n        q = query.Or(\n            [query.Term(\"content\", word, boost=weight) for word, weight in kts],\n        )\n        mask = {docnum}\n\n        return q, mask\n\n\ndef autocomplete(ix, term, limit=10):\n    with ix.reader() as reader:\n        terms = []\n        for (score, t) in reader.most_distinctive_terms(\n            \"content\",\n            number=limit,\n            prefix=term.lower(),\n        ):\n            terms.append(t)\n        return terms\n", "path": "src/documents/index.py"}], "after_files": [{"content": "import logging\nimport math\nimport os\nfrom contextlib import contextmanager\n\nfrom dateutil.parser import isoparse\nfrom django.conf import settings\nfrom documents.models import Document\nfrom whoosh import classify\nfrom whoosh import highlight\nfrom whoosh import query\nfrom whoosh.fields import BOOLEAN\nfrom whoosh.fields import DATETIME\nfrom whoosh.fields import KEYWORD\nfrom whoosh.fields import NUMERIC\nfrom whoosh.fields import Schema\nfrom whoosh.fields import TEXT\nfrom whoosh.highlight import HtmlFormatter\nfrom whoosh.index import create_in\nfrom whoosh.index import exists_in\nfrom whoosh.index import open_dir\nfrom whoosh.qparser import MultifieldParser\nfrom whoosh.qparser.dateparse import DateParserPlugin\nfrom whoosh.searching import ResultsPage\nfrom whoosh.searching import Searcher\nfrom whoosh.writing import AsyncWriter\n\nlogger = logging.getLogger(\"paperless.index\")\n\n\ndef get_schema():\n    return Schema(\n        id=NUMERIC(stored=True, unique=True),\n        title=TEXT(sortable=True),\n        content=TEXT(),\n        asn=NUMERIC(sortable=True),\n        correspondent=TEXT(sortable=True),\n        correspondent_id=NUMERIC(),\n        has_correspondent=BOOLEAN(),\n        tag=KEYWORD(commas=True, scorable=True, lowercase=True),\n        tag_id=KEYWORD(commas=True, scorable=True),\n        has_tag=BOOLEAN(),\n        type=TEXT(sortable=True),\n        type_id=NUMERIC(),\n        has_type=BOOLEAN(),\n        created=DATETIME(sortable=True),\n        modified=DATETIME(sortable=True),\n        added=DATETIME(sortable=True),\n        path=TEXT(sortable=True),\n        path_id=NUMERIC(),\n        has_path=BOOLEAN(),\n    )\n\n\ndef open_index(recreate=False):\n    try:\n        if exists_in(settings.INDEX_DIR) and not recreate:\n            return open_dir(settings.INDEX_DIR, schema=get_schema())\n    except Exception:\n        logger.exception(\"Error while opening the index, recreating.\")\n\n    if not os.path.isdir(settings.INDEX_DIR):\n        os.makedirs(settings.INDEX_DIR, exist_ok=True)\n    return create_in(settings.INDEX_DIR, get_schema())\n\n\n@contextmanager\ndef open_index_writer(optimize=False):\n    writer = AsyncWriter(open_index())\n\n    try:\n        yield writer\n    except Exception as e:\n        logger.exception(str(e))\n        writer.cancel()\n    finally:\n        writer.commit(optimize=optimize)\n\n\n@contextmanager\ndef open_index_searcher():\n    searcher = open_index().searcher()\n\n    try:\n        yield searcher\n    finally:\n        searcher.close()\n\n\ndef update_document(writer, doc):\n    tags = \",\".join([t.name for t in doc.tags.all()])\n    tags_ids = \",\".join([str(t.id) for t in doc.tags.all()])\n    writer.update_document(\n        id=doc.pk,\n        title=doc.title,\n        content=doc.content,\n        correspondent=doc.correspondent.name if doc.correspondent else None,\n        correspondent_id=doc.correspondent.id if doc.correspondent else None,\n        has_correspondent=doc.correspondent is not None,\n        tag=tags if tags else None,\n        tag_id=tags_ids if tags_ids else None,\n        has_tag=len(tags) > 0,\n        type=doc.document_type.name if doc.document_type else None,\n        type_id=doc.document_type.id if doc.document_type else None,\n        has_type=doc.document_type is not None,\n        created=doc.created,\n        added=doc.added,\n        asn=doc.archive_serial_number,\n        modified=doc.modified,\n        path=doc.storage_path.name if doc.storage_path else None,\n        path_id=doc.storage_path.id if doc.storage_path else None,\n        has_path=doc.storage_path is not None,\n    )\n\n\ndef remove_document(writer, doc):\n    remove_document_by_id(writer, doc.pk)\n\n\ndef remove_document_by_id(writer, doc_id):\n    writer.delete_by_term(\"id\", doc_id)\n\n\ndef add_or_update_document(document):\n    with open_index_writer() as writer:\n        update_document(writer, document)\n\n\ndef remove_document_from_index(document):\n    with open_index_writer() as writer:\n        remove_document(writer, document)\n\n\nclass DelayedQuery:\n    def _get_query(self):\n        raise NotImplementedError()\n\n    def _get_query_filter(self):\n        criterias = []\n        for k, v in self.query_params.items():\n            if k == \"correspondent__id\":\n                criterias.append(query.Term(\"correspondent_id\", v))\n            elif k == \"tags__id__all\":\n                for tag_id in v.split(\",\"):\n                    criterias.append(query.Term(\"tag_id\", tag_id))\n            elif k == \"tags__id__none\":\n                for tag_id in v.split(\",\"):\n                    criterias.append(query.Not(query.Term(\"tag_id\", tag_id)))\n            elif k == \"document_type__id\":\n                criterias.append(query.Term(\"type_id\", v))\n            elif k == \"correspondent__isnull\":\n                criterias.append(query.Term(\"has_correspondent\", v == \"false\"))\n            elif k == \"is_tagged\":\n                criterias.append(query.Term(\"has_tag\", v == \"true\"))\n            elif k == \"document_type__isnull\":\n                criterias.append(query.Term(\"has_type\", v == \"false\"))\n            elif k == \"created__date__lt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=None, end=isoparse(v)),\n                )\n            elif k == \"created__date__gt\":\n                criterias.append(\n                    query.DateRange(\"created\", start=isoparse(v), end=None),\n                )\n            elif k == \"added__date__gt\":\n                criterias.append(query.DateRange(\"added\", start=isoparse(v), end=None))\n            elif k == \"added__date__lt\":\n                criterias.append(query.DateRange(\"added\", start=None, end=isoparse(v)))\n            elif k == \"storage_path__id\":\n                criterias.append(query.Term(\"path_id\", v))\n            elif k == \"storage_path__isnull\":\n                criterias.append(query.Term(\"has_path\", v == \"false\"))\n\n        if len(criterias) > 0:\n            return query.And(criterias)\n        else:\n            return None\n\n    def _get_query_sortedby(self):\n        if \"ordering\" not in self.query_params:\n            return None, False\n\n        field: str = self.query_params[\"ordering\"]\n\n        sort_fields_map = {\n            \"created\": \"created\",\n            \"modified\": \"modified\",\n            \"added\": \"added\",\n            \"title\": \"title\",\n            \"correspondent__name\": \"correspondent\",\n            \"document_type__name\": \"type\",\n            \"archive_serial_number\": \"asn\",\n        }\n\n        if field.startswith(\"-\"):\n            field = field[1:]\n            reverse = True\n        else:\n            reverse = False\n\n        if field not in sort_fields_map:\n            return None, False\n        else:\n            return sort_fields_map[field], reverse\n\n    def __init__(self, searcher: Searcher, query_params, page_size):\n        self.searcher = searcher\n        self.query_params = query_params\n        self.page_size = page_size\n        self.saved_results = dict()\n        self.first_score = None\n\n    def __len__(self):\n        page = self[0:1]\n        return len(page)\n\n    def __getitem__(self, item):\n        if item.start in self.saved_results:\n            return self.saved_results[item.start]\n\n        q, mask = self._get_query()\n        sortedby, reverse = self._get_query_sortedby()\n\n        page: ResultsPage = self.searcher.search_page(\n            q,\n            mask=mask,\n            filter=self._get_query_filter(),\n            pagenum=math.floor(item.start / self.page_size) + 1,\n            pagelen=self.page_size,\n            sortedby=sortedby,\n            reverse=reverse,\n        )\n        page.results.fragmenter = highlight.ContextFragmenter(surround=50)\n        page.results.formatter = HtmlFormatter(tagname=\"span\", between=\" ... \")\n\n        if not self.first_score and len(page.results) > 0 and sortedby is None:\n            self.first_score = page.results[0].score\n\n        page.results.top_n = list(\n            map(\n                lambda hit: (\n                    (hit[0] / self.first_score) if self.first_score else None,\n                    hit[1],\n                ),\n                page.results.top_n,\n            ),\n        )\n\n        self.saved_results[item.start] = page\n\n        return page\n\n\nclass DelayedFullTextQuery(DelayedQuery):\n    def _get_query(self):\n        q_str = self.query_params[\"query\"]\n        qp = MultifieldParser(\n            [\"content\", \"title\", \"correspondent\", \"tag\", \"type\"],\n            self.searcher.ixreader.schema,\n        )\n        qp.add_plugin(DateParserPlugin())\n        q = qp.parse(q_str)\n\n        corrected = self.searcher.correct_query(q, q_str)\n        if corrected.query != q:\n            corrected.query = corrected.string\n\n        return q, None\n\n\nclass DelayedMoreLikeThisQuery(DelayedQuery):\n    def _get_query(self):\n        more_like_doc_id = int(self.query_params[\"more_like_id\"])\n        content = Document.objects.get(id=more_like_doc_id).content\n\n        docnum = self.searcher.document_number(id=more_like_doc_id)\n        kts = self.searcher.key_terms_from_text(\n            \"content\",\n            content,\n            numterms=20,\n            model=classify.Bo1Model,\n            normalize=False,\n        )\n        q = query.Or(\n            [query.Term(\"content\", word, boost=weight) for word, weight in kts],\n        )\n        mask = {docnum}\n\n        return q, mask\n\n\ndef autocomplete(ix, term, limit=10):\n    with ix.reader() as reader:\n        terms = []\n        for (score, t) in reader.most_distinctive_terms(\n            \"content\",\n            number=limit,\n            prefix=term.lower(),\n        ):\n            terms.append(t)\n        return terms\n", "path": "src/documents/index.py"}]}