We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Projects built with `htmldir` have invalid link for homepage in search results
Projects that are built with `htmldir` have invalid link in search results to homepage.
For example: https://2.python-requests.org/en/master/search/?q=%22The+Contributor%22

This is related to #5821.

We are using `processed_json.path` for the search results, but the correct path is present in `full_path`.

https://github.com/readthedocs/readthedocs.org/blob/3c04a96b3a9ea8b68ecf270670080285c73c689b/readthedocs/search/documents.py#L73-L74

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `readthedocs/search/api.py`
Content:
```
1 import itertools
2 import logging
3 
4 from django.shortcuts import get_object_or_404
5 from django.utils import timezone
6 from rest_framework import generics, serializers
7 from rest_framework.exceptions import ValidationError
8 from rest_framework.pagination import PageNumberPagination
9 
10 from readthedocs.api.v2.permissions import IsAuthorizedToViewVersion
11 from readthedocs.builds.models import Version
12 from readthedocs.projects.models import HTMLFile, Project
13 from readthedocs.search import tasks, utils
14 from readthedocs.search.faceted_search import PageSearch
15 
16 log = logging.getLogger(__name__)
17 
18 
19 class SearchPagination(PageNumberPagination):
20     page_size = 50
21     page_size_query_param = 'page_size'
22     max_page_size = 100
23 
24 
25 class PageSearchSerializer(serializers.Serializer):
26     project = serializers.CharField()
27     version = serializers.CharField()
28     title = serializers.CharField()
29     path = serializers.CharField()
30     link = serializers.SerializerMethodField()
31     highlight = serializers.SerializerMethodField()
32     inner_hits = serializers.SerializerMethodField()
33 
34     def get_link(self, obj):
35         projects_url = self.context.get('projects_url')
36         if projects_url:
37             docs_url = projects_url[obj.project]
38             return docs_url + obj.path
39 
40     def get_highlight(self, obj):
41         highlight = getattr(obj.meta, 'highlight', None)
42         if highlight:
43             ret = highlight.to_dict()
44             log.debug('API Search highlight [Page title]: %s', ret)
45             return ret
46 
47     def get_inner_hits(self, obj):
48         inner_hits = getattr(obj.meta, 'inner_hits', None)
49         if inner_hits:
50             sections = inner_hits.sections or []
51             domains = inner_hits.domains or []
52             all_results = itertools.chain(sections, domains)
53 
54             sorted_results = utils._get_sorted_results(
55                 results=all_results,
56                 source_key='_source',
57             )
58 
59             log.debug('[API] Sorted Results: %s', sorted_results)
60             return sorted_results
61 
62 
63 class PageSearchAPIView(generics.ListAPIView):
64 
65     """
66     Main entry point to perform a search using Elasticsearch.
67 
68     Required query params:
69     - q (search term)
70     - project
71     - version
72 
73     .. note::
74 
75        The methods `_get_project` and `_get_version`
76        are called many times, so a basic cache is implemented.
77     """
78 
79     permission_classes = [IsAuthorizedToViewVersion]
80     pagination_class = SearchPagination
81     serializer_class = PageSearchSerializer
82 
83     def _get_project(self):
84         cache_key = '_cached_project'
85         project = getattr(self, cache_key, None)
86 
87         if not project:
88             project_slug = self.request.GET.get('project', None)
89             project = get_object_or_404(Project, slug=project_slug)
90             setattr(self, cache_key, project)
91 
92         return project
93 
94     def _get_version(self):
95         cache_key = '_cached_version'
96         version = getattr(self, cache_key, None)
97 
98         if not version:
99             version_slug = self.request.GET.get('version', None)
100             project = self._get_project()
101             version = get_object_or_404(
102                 project.versions.all(),
103                 slug=version_slug,
104             )
105             setattr(self, cache_key, version)
106 
107         return version
108 
109     def get_queryset(self):
110         """
111         Return Elasticsearch DSL Search object instead of Django Queryset.
112 
113         Django Queryset and elasticsearch-dsl ``Search`` object is similar pattern.
114         So for searching, its possible to return ``Search`` object instead of queryset.
115         The ``filter_backends`` and ``pagination_class`` is compatible with ``Search``
116         """
117         # Validate all the required params are there
118         self.validate_query_params()
119         query = self.request.query_params.get('q', '')
120         kwargs = {'filter_by_user': False, 'filters': {}}
121         kwargs['filters']['project'] = [p.slug for p in self.get_all_projects()]
122         kwargs['filters']['version'] = self._get_version().slug
123 
124         # Check to avoid searching all projects in case these filters are empty.
125         if not kwargs['filters']['project']:
126             log.info("Unable to find a project to search")
127             return HTMLFile.objects.none()
128         if not kwargs['filters']['version']:
129             log.info("Unable to find a version to search")
130             return HTMLFile.objects.none()
131 
132         user = self.request.user
133         queryset = PageSearch(
134             query=query, user=user, **kwargs
135         )
136         return queryset
137 
138     def validate_query_params(self):
139         """
140         Validate all required query params are passed on the request.
141 
142         Query params required are: ``q``, ``project`` and ``version``.
143 
144         :rtype: None
145 
146         :raises: ValidationError if one of them is missing.
147         """
148         required_query_params = {'q', 'project', 'version'}  # python `set` literal is `{}`
149         request_params = set(self.request.query_params.keys())
150         missing_params = required_query_params - request_params
151         if missing_params:
152             errors = {}
153             for param in missing_params:
154                 errors[param] = ["This query param is required"]
155 
156             raise ValidationError(errors)
157 
158     def get_serializer_context(self):
159         context = super().get_serializer_context()
160         context['projects_url'] = self.get_all_projects_url()
161         return context
162 
163     def get_all_projects(self):
164         """
165         Return a list of the project itself and all its subprojects the user has permissions over.
166 
167         :rtype: list
168         """
169         main_version = self._get_version()
170         main_project = self._get_project()
171 
172         all_projects = [main_project]
173 
174         subprojects = Project.objects.filter(
175             superprojects__parent_id=main_project.id,
176         )
177         for project in subprojects:
178             version = (
179                 Version.internal
180                 .public(user=self.request.user, project=project, include_hidden=False)
181                 .filter(slug=main_version.slug)
182                 .first()
183             )
184             if version:
185                 all_projects.append(version.project)
186         return all_projects
187 
188     def get_all_projects_url(self):
189         """
190         Return a dict containing the project slug and its version URL.
191 
192         The dictionary contains the project and its subprojects . Each project's
193         slug is used as a key and the documentation URL for that project and
194         version as the value.
195 
196         Example:
197 
198         {
199             "requests": "https://requests.readthedocs.io/en/latest/",
200             "requests-oauth": "https://requests-oauth.readthedocs.io/en/latest/",
201         }
202 
203         :rtype: dict
204         """
205         all_projects = self.get_all_projects()
206         version_slug = self._get_version().slug
207         projects_url = {}
208         for project in all_projects:
209             projects_url[project.slug] = project.get_docs_url(version_slug=version_slug)
210         return projects_url
211 
212     def list(self, request, *args, **kwargs):
213         """Overriding ``list`` method to record query in database."""
214 
215         response = super().list(request, *args, **kwargs)
216 
217         project_slug = self._get_project().slug
218         version_slug = self._get_version().slug
219         total_results = response.data.get('count', 0)
220         time = timezone.now()
221 
222         query = self.request.query_params.get('q', '')
223         query = query.lower().strip()
224 
225         # record the search query with a celery task
226         tasks.record_search_query.delay(
227             project_slug,
228             version_slug,
229             query,
230             total_results,
231             time.isoformat(),
232         )
233 
234         return response
235
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/readthedocs/search/api.py b/readthedocs/search/api.py
--- a/readthedocs/search/api.py
+++ b/readthedocs/search/api.py
@@ -1,5 +1,6 @@
 import itertools
 import logging
+import re
 
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
@@ -9,6 +10,7 @@
 
 from readthedocs.api.v2.permissions import IsAuthorizedToViewVersion
 from readthedocs.builds.models import Version
+from readthedocs.projects.constants import MKDOCS, SPHINX_HTMLDIR
 from readthedocs.projects.models import HTMLFile, Project
 from readthedocs.search import tasks, utils
 from readthedocs.search.faceted_search import PageSearch
@@ -27,15 +29,28 @@
     version = serializers.CharField()
     title = serializers.CharField()
     path = serializers.CharField()
+    full_path = serializers.CharField()
     link = serializers.SerializerMethodField()
     highlight = serializers.SerializerMethodField()
     inner_hits = serializers.SerializerMethodField()
 
     def get_link(self, obj):
-        projects_url = self.context.get('projects_url')
-        if projects_url:
-            docs_url = projects_url[obj.project]
-            return docs_url + obj.path
+        project_data = self.context['projects_data'].get(obj.project)
+        if not project_data:
+            return None
+
+        docs_url, doctype = project_data
+        path = obj.full_path
+
+        # Generate an appropriate link for the doctypes that use htmldir,
+        # and always end it with / so it goes directly to proxito.
+        if doctype in {SPHINX_HTMLDIR, MKDOCS}:
+            new_path = re.sub('(^|/)index.html$', '/', path)
+            # docs_url already ends with /,
+            # so path doesn't need to start with /.
+            path = new_path.lstrip('/')
+
+        return docs_url + path
 
     def get_highlight(self, obj):
         highlight = getattr(obj.meta, 'highlight', None)
@@ -157,7 +172,7 @@
 
     def get_serializer_context(self):
         context = super().get_serializer_context()
-        context['projects_url'] = self.get_all_projects_url()
+        context['projects_data'] = self.get_all_projects_data()
         return context
 
     def get_all_projects(self):
@@ -185,29 +200,44 @@
                 all_projects.append(version.project)
         return all_projects
 
-    def get_all_projects_url(self):
+    def get_all_projects_data(self):
         """
-        Return a dict containing the project slug and its version URL.
-
-        The dictionary contains the project and its subprojects . Each project's
-        slug is used as a key and the documentation URL for that project and
-        version as the value.
+        Return a dict containing the project slug and its version URL and version's doctype.
 
-        Example:
+        The dictionary contains the project and its subprojects. Each project's
+        slug is used as a key and a tuple with the documentation URL and doctype
+        from the version. Example:
 
         {
-            "requests": "https://requests.readthedocs.io/en/latest/",
-            "requests-oauth": "https://requests-oauth.readthedocs.io/en/latest/",
+            "requests": (
+                "https://requests.readthedocs.io/en/latest/",
+                "sphinx",
+            ),
+            "requests-oauth": (
+                "https://requests-oauth.readthedocs.io/en/latest/",
+                "sphinx_htmldir",
+            ),
         }
 
         :rtype: dict
         """
         all_projects = self.get_all_projects()
         version_slug = self._get_version().slug
-        projects_url = {}
+        project_urls = {}
         for project in all_projects:
-            projects_url[project.slug] = project.get_docs_url(version_slug=version_slug)
-        return projects_url
+            project_urls[project.slug] = project.get_docs_url(version_slug=version_slug)
+
+        versions_doctype = (
+            Version.objects
+            .filter(project__slug__in=project_urls.keys(), slug=version_slug)
+            .values_list('project__slug', 'documentation_type')
+        )
+
+        projects_data = {
+            project_slug: (project_urls[project_slug], doctype)
+            for project_slug, doctype in versions_doctype
+        }
+        return projects_data
 
     def list(self, request, *args, **kwargs):
         """Overriding ``list`` method to record query in database."""

{"golden_diff": "diff --git a/readthedocs/search/api.py b/readthedocs/search/api.py\n--- a/readthedocs/search/api.py\n+++ b/readthedocs/search/api.py\n@@ -1,5 +1,6 @@\n import itertools\n import logging\n+import re\n \n from django.shortcuts import get_object_or_404\n from django.utils import timezone\n@@ -9,6 +10,7 @@\n \n from readthedocs.api.v2.permissions import IsAuthorizedToViewVersion\n from readthedocs.builds.models import Version\n+from readthedocs.projects.constants import MKDOCS, SPHINX_HTMLDIR\n from readthedocs.projects.models import HTMLFile, Project\n from readthedocs.search import tasks, utils\n from readthedocs.search.faceted_search import PageSearch\n@@ -27,15 +29,28 @@\n     version = serializers.CharField()\n     title = serializers.CharField()\n     path = serializers.CharField()\n+    full_path = serializers.CharField()\n     link = serializers.SerializerMethodField()\n     highlight = serializers.SerializerMethodField()\n     inner_hits = serializers.SerializerMethodField()\n \n     def get_link(self, obj):\n-        projects_url = self.context.get('projects_url')\n-        if projects_url:\n-            docs_url = projects_url[obj.project]\n-            return docs_url + obj.path\n+        project_data = self.context['projects_data'].get(obj.project)\n+        if not project_data:\n+            return None\n+\n+        docs_url, doctype = project_data\n+        path = obj.full_path\n+\n+        # Generate an appropriate link for the doctypes that use htmldir,\n+        # and always end it with / so it goes directly to proxito.\n+        if doctype in {SPHINX_HTMLDIR, MKDOCS}:\n+            new_path = re.sub('(^|/)index.html$', '/', path)\n+            # docs_url already ends with /,\n+            # so path doesn't need to start with /.\n+            path = new_path.lstrip('/')\n+\n+        return docs_url + path\n \n     def get_highlight(self, obj):\n         highlight = getattr(obj.meta, 'highlight', None)\n@@ -157,7 +172,7 @@\n \n     def get_serializer_context(self):\n         context = super().get_serializer_context()\n-        context['projects_url'] = self.get_all_projects_url()\n+        context['projects_data'] = self.get_all_projects_data()\n         return context\n \n     def get_all_projects(self):\n@@ -185,29 +200,44 @@\n                 all_projects.append(version.project)\n         return all_projects\n \n-    def get_all_projects_url(self):\n+    def get_all_projects_data(self):\n         \"\"\"\n-        Return a dict containing the project slug and its version URL.\n-\n-        The dictionary contains the project and its subprojects . Each project's\n-        slug is used as a key and the documentation URL for that project and\n-        version as the value.\n+        Return a dict containing the project slug and its version URL and version's doctype.\n \n-        Example:\n+        The dictionary contains the project and its subprojects. Each project's\n+        slug is used as a key and a tuple with the documentation URL and doctype\n+        from the version. Example:\n \n         {\n-            \"requests\": \"https://requests.readthedocs.io/en/latest/\",\n-            \"requests-oauth\": \"https://requests-oauth.readthedocs.io/en/latest/\",\n+            \"requests\": (\n+                \"https://requests.readthedocs.io/en/latest/\",\n+                \"sphinx\",\n+            ),\n+            \"requests-oauth\": (\n+                \"https://requests-oauth.readthedocs.io/en/latest/\",\n+                \"sphinx_htmldir\",\n+            ),\n         }\n \n         :rtype: dict\n         \"\"\"\n         all_projects = self.get_all_projects()\n         version_slug = self._get_version().slug\n-        projects_url = {}\n+        project_urls = {}\n         for project in all_projects:\n-            projects_url[project.slug] = project.get_docs_url(version_slug=version_slug)\n-        return projects_url\n+            project_urls[project.slug] = project.get_docs_url(version_slug=version_slug)\n+\n+        versions_doctype = (\n+            Version.objects\n+            .filter(project__slug__in=project_urls.keys(), slug=version_slug)\n+            .values_list('project__slug', 'documentation_type')\n+        )\n+\n+        projects_data = {\n+            project_slug: (project_urls[project_slug], doctype)\n+            for project_slug, doctype in versions_doctype\n+        }\n+        return projects_data\n \n     def list(self, request, *args, **kwargs):\n         \"\"\"Overriding ``list`` method to record query in database.\"\"\"\n", "issue": "Projects built with `htmldir` have invalid link for homepage in search results\nProjects that are built with `htmldir` have invalid link in search results to homepage.\r\nFor example: https://2.python-requests.org/en/master/search/?q=%22The+Contributor%22\r\n\r\nThis is related to #5821.\r\n\r\nWe are using `processed_json.path` for the search results, but the correct path is present in `full_path`.\r\n\r\nhttps://github.com/readthedocs/readthedocs.org/blob/3c04a96b3a9ea8b68ecf270670080285c73c689b/readthedocs/search/documents.py#L73-L74\n", "before_files": [{"content": "import itertools\nimport logging\n\nfrom django.shortcuts import get_object_or_404\nfrom django.utils import timezone\nfrom rest_framework import generics, serializers\nfrom rest_framework.exceptions import ValidationError\nfrom rest_framework.pagination import PageNumberPagination\n\nfrom readthedocs.api.v2.permissions import IsAuthorizedToViewVersion\nfrom readthedocs.builds.models import Version\nfrom readthedocs.projects.models import HTMLFile, Project\nfrom readthedocs.search import tasks, utils\nfrom readthedocs.search.faceted_search import PageSearch\n\nlog = logging.getLogger(__name__)\n\n\nclass SearchPagination(PageNumberPagination):\n    page_size = 50\n    page_size_query_param = 'page_size'\n    max_page_size = 100\n\n\nclass PageSearchSerializer(serializers.Serializer):\n    project = serializers.CharField()\n    version = serializers.CharField()\n    title = serializers.CharField()\n    path = serializers.CharField()\n    link = serializers.SerializerMethodField()\n    highlight = serializers.SerializerMethodField()\n    inner_hits = serializers.SerializerMethodField()\n\n    def get_link(self, obj):\n        projects_url = self.context.get('projects_url')\n        if projects_url:\n            docs_url = projects_url[obj.project]\n            return docs_url + obj.path\n\n    def get_highlight(self, obj):\n        highlight = getattr(obj.meta, 'highlight', None)\n        if highlight:\n            ret = highlight.to_dict()\n            log.debug('API Search highlight [Page title]: %s', ret)\n            return ret\n\n    def get_inner_hits(self, obj):\n        inner_hits = getattr(obj.meta, 'inner_hits', None)\n        if inner_hits:\n            sections = inner_hits.sections or []\n            domains = inner_hits.domains or []\n            all_results = itertools.chain(sections, domains)\n\n            sorted_results = utils._get_sorted_results(\n                results=all_results,\n                source_key='_source',\n            )\n\n            log.debug('[API] Sorted Results: %s', sorted_results)\n            return sorted_results\n\n\nclass PageSearchAPIView(generics.ListAPIView):\n\n    \"\"\"\n    Main entry point to perform a search using Elasticsearch.\n\n    Required query params:\n    - q (search term)\n    - project\n    - version\n\n    .. note::\n\n       The methods `_get_project` and `_get_version`\n       are called many times, so a basic cache is implemented.\n    \"\"\"\n\n    permission_classes = [IsAuthorizedToViewVersion]\n    pagination_class = SearchPagination\n    serializer_class = PageSearchSerializer\n\n    def _get_project(self):\n        cache_key = '_cached_project'\n        project = getattr(self, cache_key, None)\n\n        if not project:\n            project_slug = self.request.GET.get('project', None)\n            project = get_object_or_404(Project, slug=project_slug)\n            setattr(self, cache_key, project)\n\n        return project\n\n    def _get_version(self):\n        cache_key = '_cached_version'\n        version = getattr(self, cache_key, None)\n\n        if not version:\n            version_slug = self.request.GET.get('version', None)\n            project = self._get_project()\n            version = get_object_or_404(\n                project.versions.all(),\n                slug=version_slug,\n            )\n            setattr(self, cache_key, version)\n\n        return version\n\n    def get_queryset(self):\n        \"\"\"\n        Return Elasticsearch DSL Search object instead of Django Queryset.\n\n        Django Queryset and elasticsearch-dsl ``Search`` object is similar pattern.\n        So for searching, its possible to return ``Search`` object instead of queryset.\n        The ``filter_backends`` and ``pagination_class`` is compatible with ``Search``\n        \"\"\"\n        # Validate all the required params are there\n        self.validate_query_params()\n        query = self.request.query_params.get('q', '')\n        kwargs = {'filter_by_user': False, 'filters': {}}\n        kwargs['filters']['project'] = [p.slug for p in self.get_all_projects()]\n        kwargs['filters']['version'] = self._get_version().slug\n\n        # Check to avoid searching all projects in case these filters are empty.\n        if not kwargs['filters']['project']:\n            log.info(\"Unable to find a project to search\")\n            return HTMLFile.objects.none()\n        if not kwargs['filters']['version']:\n            log.info(\"Unable to find a version to search\")\n            return HTMLFile.objects.none()\n\n        user = self.request.user\n        queryset = PageSearch(\n            query=query, user=user, **kwargs\n        )\n        return queryset\n\n    def validate_query_params(self):\n        \"\"\"\n        Validate all required query params are passed on the request.\n\n        Query params required are: ``q``, ``project`` and ``version``.\n\n        :rtype: None\n\n        :raises: ValidationError if one of them is missing.\n        \"\"\"\n        required_query_params = {'q', 'project', 'version'}  # python `set` literal is `{}`\n        request_params = set(self.request.query_params.keys())\n        missing_params = required_query_params - request_params\n        if missing_params:\n            errors = {}\n            for param in missing_params:\n                errors[param] = [\"This query param is required\"]\n\n            raise ValidationError(errors)\n\n    def get_serializer_context(self):\n        context = super().get_serializer_context()\n        context['projects_url'] = self.get_all_projects_url()\n        return context\n\n    def get_all_projects(self):\n        \"\"\"\n        Return a list of the project itself and all its subprojects the user has permissions over.\n\n        :rtype: list\n        \"\"\"\n        main_version = self._get_version()\n        main_project = self._get_project()\n\n        all_projects = [main_project]\n\n        subprojects = Project.objects.filter(\n            superprojects__parent_id=main_project.id,\n        )\n        for project in subprojects:\n            version = (\n                Version.internal\n                .public(user=self.request.user, project=project, include_hidden=False)\n                .filter(slug=main_version.slug)\n                .first()\n            )\n            if version:\n                all_projects.append(version.project)\n        return all_projects\n\n    def get_all_projects_url(self):\n        \"\"\"\n        Return a dict containing the project slug and its version URL.\n\n        The dictionary contains the project and its subprojects . Each project's\n        slug is used as a key and the documentation URL for that project and\n        version as the value.\n\n        Example:\n\n        {\n            \"requests\": \"https://requests.readthedocs.io/en/latest/\",\n            \"requests-oauth\": \"https://requests-oauth.readthedocs.io/en/latest/\",\n        }\n\n        :rtype: dict\n        \"\"\"\n        all_projects = self.get_all_projects()\n        version_slug = self._get_version().slug\n        projects_url = {}\n        for project in all_projects:\n            projects_url[project.slug] = project.get_docs_url(version_slug=version_slug)\n        return projects_url\n\n    def list(self, request, *args, **kwargs):\n        \"\"\"Overriding ``list`` method to record query in database.\"\"\"\n\n        response = super().list(request, *args, **kwargs)\n\n        project_slug = self._get_project().slug\n        version_slug = self._get_version().slug\n        total_results = response.data.get('count', 0)\n        time = timezone.now()\n\n        query = self.request.query_params.get('q', '')\n        query = query.lower().strip()\n\n        # record the search query with a celery task\n        tasks.record_search_query.delay(\n            project_slug,\n            version_slug,\n            query,\n            total_results,\n            time.isoformat(),\n        )\n\n        return response\n", "path": "readthedocs/search/api.py"}], "after_files": [{"content": "import itertools\nimport logging\nimport re\n\nfrom django.shortcuts import get_object_or_404\nfrom django.utils import timezone\nfrom rest_framework import generics, serializers\nfrom rest_framework.exceptions import ValidationError\nfrom rest_framework.pagination import PageNumberPagination\n\nfrom readthedocs.api.v2.permissions import IsAuthorizedToViewVersion\nfrom readthedocs.builds.models import Version\nfrom readthedocs.projects.constants import MKDOCS, SPHINX_HTMLDIR\nfrom readthedocs.projects.models import HTMLFile, Project\nfrom readthedocs.search import tasks, utils\nfrom readthedocs.search.faceted_search import PageSearch\n\nlog = logging.getLogger(__name__)\n\n\nclass SearchPagination(PageNumberPagination):\n    page_size = 50\n    page_size_query_param = 'page_size'\n    max_page_size = 100\n\n\nclass PageSearchSerializer(serializers.Serializer):\n    project = serializers.CharField()\n    version = serializers.CharField()\n    title = serializers.CharField()\n    path = serializers.CharField()\n    full_path = serializers.CharField()\n    link = serializers.SerializerMethodField()\n    highlight = serializers.SerializerMethodField()\n    inner_hits = serializers.SerializerMethodField()\n\n    def get_link(self, obj):\n        project_data = self.context['projects_data'].get(obj.project)\n        if not project_data:\n            return None\n\n        docs_url, doctype = project_data\n        path = obj.full_path\n\n        # Generate an appropriate link for the doctypes that use htmldir,\n        # and always end it with / so it goes directly to proxito.\n        if doctype in {SPHINX_HTMLDIR, MKDOCS}:\n            new_path = re.sub('(^|/)index.html$', '/', path)\n            # docs_url already ends with /,\n            # so path doesn't need to start with /.\n            path = new_path.lstrip('/')\n\n        return docs_url + path\n\n    def get_highlight(self, obj):\n        highlight = getattr(obj.meta, 'highlight', None)\n        if highlight:\n            ret = highlight.to_dict()\n            log.debug('API Search highlight [Page title]: %s', ret)\n            return ret\n\n    def get_inner_hits(self, obj):\n        inner_hits = getattr(obj.meta, 'inner_hits', None)\n        if inner_hits:\n            sections = inner_hits.sections or []\n            domains = inner_hits.domains or []\n            all_results = itertools.chain(sections, domains)\n\n            sorted_results = utils._get_sorted_results(\n                results=all_results,\n                source_key='_source',\n            )\n\n            log.debug('[API] Sorted Results: %s', sorted_results)\n            return sorted_results\n\n\nclass PageSearchAPIView(generics.ListAPIView):\n\n    \"\"\"\n    Main entry point to perform a search using Elasticsearch.\n\n    Required query params:\n    - q (search term)\n    - project\n    - version\n\n    .. note::\n\n       The methods `_get_project` and `_get_version`\n       are called many times, so a basic cache is implemented.\n    \"\"\"\n\n    permission_classes = [IsAuthorizedToViewVersion]\n    pagination_class = SearchPagination\n    serializer_class = PageSearchSerializer\n\n    def _get_project(self):\n        cache_key = '_cached_project'\n        project = getattr(self, cache_key, None)\n\n        if not project:\n            project_slug = self.request.GET.get('project', None)\n            project = get_object_or_404(Project, slug=project_slug)\n            setattr(self, cache_key, project)\n\n        return project\n\n    def _get_version(self):\n        cache_key = '_cached_version'\n        version = getattr(self, cache_key, None)\n\n        if not version:\n            version_slug = self.request.GET.get('version', None)\n            project = self._get_project()\n            version = get_object_or_404(\n                project.versions.all(),\n                slug=version_slug,\n            )\n            setattr(self, cache_key, version)\n\n        return version\n\n    def get_queryset(self):\n        \"\"\"\n        Return Elasticsearch DSL Search object instead of Django Queryset.\n\n        Django Queryset and elasticsearch-dsl ``Search`` object is similar pattern.\n        So for searching, its possible to return ``Search`` object instead of queryset.\n        The ``filter_backends`` and ``pagination_class`` is compatible with ``Search``\n        \"\"\"\n        # Validate all the required params are there\n        self.validate_query_params()\n        query = self.request.query_params.get('q', '')\n        kwargs = {'filter_by_user': False, 'filters': {}}\n        kwargs['filters']['project'] = [p.slug for p in self.get_all_projects()]\n        kwargs['filters']['version'] = self._get_version().slug\n\n        # Check to avoid searching all projects in case these filters are empty.\n        if not kwargs['filters']['project']:\n            log.info(\"Unable to find a project to search\")\n            return HTMLFile.objects.none()\n        if not kwargs['filters']['version']:\n            log.info(\"Unable to find a version to search\")\n            return HTMLFile.objects.none()\n\n        user = self.request.user\n        queryset = PageSearch(\n            query=query, user=user, **kwargs\n        )\n        return queryset\n\n    def validate_query_params(self):\n        \"\"\"\n        Validate all required query params are passed on the request.\n\n        Query params required are: ``q``, ``project`` and ``version``.\n\n        :rtype: None\n\n        :raises: ValidationError if one of them is missing.\n        \"\"\"\n        required_query_params = {'q', 'project', 'version'}  # python `set` literal is `{}`\n        request_params = set(self.request.query_params.keys())\n        missing_params = required_query_params - request_params\n        if missing_params:\n            errors = {}\n            for param in missing_params:\n                errors[param] = [\"This query param is required\"]\n\n            raise ValidationError(errors)\n\n    def get_serializer_context(self):\n        context = super().get_serializer_context()\n        context['projects_data'] = self.get_all_projects_data()\n        return context\n\n    def get_all_projects(self):\n        \"\"\"\n        Return a list of the project itself and all its subprojects the user has permissions over.\n\n        :rtype: list\n        \"\"\"\n        main_version = self._get_version()\n        main_project = self._get_project()\n\n        all_projects = [main_project]\n\n        subprojects = Project.objects.filter(\n            superprojects__parent_id=main_project.id,\n        )\n        for project in subprojects:\n            version = (\n                Version.internal\n                .public(user=self.request.user, project=project, include_hidden=False)\n                .filter(slug=main_version.slug)\n                .first()\n            )\n            if version:\n                all_projects.append(version.project)\n        return all_projects\n\n    def get_all_projects_data(self):\n        \"\"\"\n        Return a dict containing the project slug and its version URL and version's doctype.\n\n        The dictionary contains the project and its subprojects. Each project's\n        slug is used as a key and a tuple with the documentation URL and doctype\n        from the version. Example:\n\n        {\n            \"requests\": (\n                \"https://requests.readthedocs.io/en/latest/\",\n                \"sphinx\",\n            ),\n            \"requests-oauth\": (\n                \"https://requests-oauth.readthedocs.io/en/latest/\",\n                \"sphinx_htmldir\",\n            ),\n        }\n\n        :rtype: dict\n        \"\"\"\n        all_projects = self.get_all_projects()\n        version_slug = self._get_version().slug\n        project_urls = {}\n        for project in all_projects:\n            project_urls[project.slug] = project.get_docs_url(version_slug=version_slug)\n\n        versions_doctype = (\n            Version.objects\n            .filter(project__slug__in=project_urls.keys(), slug=version_slug)\n            .values_list('project__slug', 'documentation_type')\n        )\n\n        projects_data = {\n            project_slug: (project_urls[project_slug], doctype)\n            for project_slug, doctype in versions_doctype\n        }\n        return projects_data\n\n    def list(self, request, *args, **kwargs):\n        \"\"\"Overriding ``list`` method to record query in database.\"\"\"\n\n        response = super().list(request, *args, **kwargs)\n\n        project_slug = self._get_project().slug\n        version_slug = self._get_version().slug\n        total_results = response.data.get('count', 0)\n        time = timezone.now()\n\n        query = self.request.query_params.get('q', '')\n        query = query.lower().strip()\n\n        # record the search query with a celery task\n        tasks.record_search_query.delay(\n            project_slug,\n            version_slug,\n            query,\n            total_results,\n            time.isoformat(),\n        )\n\n        return response\n", "path": "readthedocs/search/api.py"}]}