We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Can't use `solr_user` and `solr_password` in Python 3
**CKAN version**
2.9 

**Describe the bug**
When setting the `solr_user` and `solr_password` config options running Python 3 you get a startup error:

```
  File "/usr/lib/ckan/src/ckan/ckan/lib/search/__init__.py", line 309, in check_solr_schema_version
    res = _get_schema_from_solr(SOLR_SCHEMA_FILE_OFFSET_MANAGED)
  File "/usr/lib/ckan/src/ckan/ckan/lib/search/__init__.py", line 262, in _get_schema_from_solr
    http_auth = 'Basic ' + http_auth.encode('base64').strip()
LookupError: 'base64' is not a text encoding; use codecs.encode() to handle arbitrary codecs
```


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `ckan/lib/search/__init__.py`
Content:
```
1 # encoding: utf-8
2 
3 from __future__ import print_function
4 import logging
5 import sys
6 import cgitb
7 import warnings
8 import xml.dom.minidom
9 
10 import requests
11 
12 from ckan.common import asbool, config
13 import ckan.model as model
14 import ckan.plugins as p
15 import ckan.logic as logic
16 
17 from ckan.lib.search.common import (
18     SearchIndexError, SearchError, SearchQueryError,
19     make_connection, is_available, SolrSettings
20     )
21 from ckan.lib.search.index import PackageSearchIndex, NoopSearchIndex
22 from ckan.lib.search.query import (
23     TagSearchQuery, ResourceSearchQuery, PackageSearchQuery,
24     QueryOptions, convert_legacy_parameters_to_solr
25 )
26 
27 
28 log = logging.getLogger(__name__)
29 
30 
31 def text_traceback():
32     with warnings.catch_warnings():
33         warnings.simplefilter("ignore")
34         res = 'the original traceback:'.join(
35             cgitb.text(sys.exc_info()).split('the original traceback:')[1:]
36         ).strip()
37     return res
38 
39 
40 SUPPORTED_SCHEMA_VERSIONS = ['2.8', '2.9']
41 
42 DEFAULT_OPTIONS = {
43     'limit': 20,
44     'offset': 0,
45     # about presenting the results
46     'order_by': 'rank',
47     'return_objects': False,
48     'ref_entity_with_attr': 'name',
49     'all_fields': False,
50     'search_tags': True,
51     'callback': None,  # simply passed through
52 }
53 
54 _INDICES = {
55     'package': PackageSearchIndex
56 }
57 
58 _QUERIES = {
59     'tag': TagSearchQuery,
60     'resource': ResourceSearchQuery,
61     'package': PackageSearchQuery
62 }
63 
64 SOLR_SCHEMA_FILE_OFFSET_MANAGED = '/schema?wt=schema.xml'
65 SOLR_SCHEMA_FILE_OFFSET_CLASSIC = '/admin/file/?file=schema.xml'
66 
67 
68 def _normalize_type(_type):
69     if isinstance(_type, model.domain_object.DomainObject):
70         _type = _type.__class__
71     if isinstance(_type, type):
72         _type = _type.__name__
73     return _type.strip().lower()
74 
75 
76 def index_for(_type):
77     """ Get a SearchIndex instance sub-class suitable for
78         the specified type. """
79     try:
80         _type_n = _normalize_type(_type)
81         return _INDICES[_type_n]()
82     except KeyError as ke:
83         log.warn("Unknown search type: %s" % _type)
84         return NoopSearchIndex()
85 
86 
87 def query_for(_type):
88     """ Get a SearchQuery instance sub-class suitable for the specified
89         type. """
90     try:
91         _type_n = _normalize_type(_type)
92         return _QUERIES[_type_n]()
93     except KeyError as ke:
94         raise SearchError("Unknown search type: %s" % _type)
95 
96 
97 def dispatch_by_operation(entity_type, entity, operation):
98     """Call the appropriate index method for a given notification."""
99     try:
100         index = index_for(entity_type)
101         if operation == model.domain_object.DomainObjectOperation.new:
102             index.insert_dict(entity)
103         elif operation == model.domain_object.DomainObjectOperation.changed:
104             index.update_dict(entity)
105         elif operation == model.domain_object.DomainObjectOperation.deleted:
106             index.remove_dict(entity)
107         else:
108             log.warn("Unknown operation: %s" % operation)
109     except Exception as ex:
110         log.exception(ex)
111         # we really need to know about any exceptions, so reraise
112         # (see #1172)
113         raise
114 
115 
116 class SynchronousSearchPlugin(p.SingletonPlugin):
117     """Update the search index automatically."""
118     p.implements(p.IDomainObjectModification, inherit=True)
119 
120     def notify(self, entity, operation):
121         if (not isinstance(entity, model.Package) or
122                 not asbool(config.get('ckan.search.automatic_indexing', True))):
123             return
124         if operation != model.domain_object.DomainObjectOperation.deleted:
125             dispatch_by_operation(
126                 entity.__class__.__name__,
127                 logic.get_action('package_show')(
128                     {'model': model, 'ignore_auth': True, 'validate': False,
129                      'use_cache': False},
130                     {'id': entity.id}),
131                 operation
132             )
133         elif operation == model.domain_object.DomainObjectOperation.deleted:
134             dispatch_by_operation(entity.__class__.__name__,
135                                   {'id': entity.id}, operation)
136         else:
137             log.warn("Discarded Sync. indexing for: %s" % entity)
138 
139 
140 def rebuild(package_id=None, only_missing=False, force=False, refresh=False,
141             defer_commit=False, package_ids=None, quiet=False):
142     '''
143         Rebuilds the search index.
144 
145         If a dataset id is provided, only this dataset will be reindexed.
146         When reindexing all datasets, if only_missing is True, only the
147         datasets not already indexed will be processed. If force equals
148         True, if an exception is found, the exception will be logged, but
149         the process will carry on.
150     '''
151     log.info("Rebuilding search index...")
152 
153     package_index = index_for(model.Package)
154     context = {'model': model, 'ignore_auth': True, 'validate': False,
155         'use_cache': False}
156 
157     if package_id:
158         pkg_dict = logic.get_action('package_show')(context,
159             {'id': package_id})
160         log.info('Indexing just package %r...', pkg_dict['name'])
161         package_index.remove_dict(pkg_dict)
162         package_index.insert_dict(pkg_dict)
163     elif package_ids is not None:
164         for package_id in package_ids:
165             pkg_dict = logic.get_action('package_show')(context,
166                 {'id': package_id})
167             log.info('Indexing just package %r...', pkg_dict['name'])
168             package_index.update_dict(pkg_dict, True)
169     else:
170         package_ids = [r[0] for r in model.Session.query(model.Package.id).
171                        filter(model.Package.state != 'deleted').all()]
172         if only_missing:
173             log.info('Indexing only missing packages...')
174             package_query = query_for(model.Package)
175             indexed_pkg_ids = set(package_query.get_all_entity_ids(
176                 max_results=len(package_ids)))
177             # Packages not indexed
178             package_ids = set(package_ids) - indexed_pkg_ids
179 
180             if len(package_ids) == 0:
181                 log.info('All datasets are already indexed')
182                 return
183         else:
184             log.info('Rebuilding the whole index...')
185             # When refreshing, the index is not previously cleared
186             if not refresh:
187                 package_index.clear()
188 
189         total_packages = len(package_ids)
190         for counter, pkg_id in enumerate(package_ids):
191             if not quiet:
192                 sys.stdout.write(
193                     "\rIndexing dataset {0}/{1}".format(
194                         counter +1, total_packages)
195                 )
196                 sys.stdout.flush()
197             try:
198                 package_index.update_dict(
199                     logic.get_action('package_show')(context,
200                         {'id': pkg_id}
201                     ),
202                     defer_commit
203                 )
204             except Exception as e:
205                 log.error(u'Error while indexing dataset %s: %s' %
206                           (pkg_id, repr(e)))
207                 if force:
208                     log.error(text_traceback())
209                     continue
210                 else:
211                     raise
212 
213     model.Session.commit()
214     log.info('Finished rebuilding search index.')
215 
216 
217 def commit():
218     package_index = index_for(model.Package)
219     package_index.commit()
220     log.info('Commited pending changes on the search index')
221 
222 
223 def check():
224     package_query = query_for(model.Package)
225 
226     log.debug("Checking packages search index...")
227     pkgs_q = model.Session.query(model.Package).filter_by(
228         state=model.State.ACTIVE)
229     pkgs = {pkg.id for pkg in pkgs_q}
230     indexed_pkgs = set(package_query.get_all_entity_ids(max_results=len(pkgs)))
231     pkgs_not_indexed = pkgs - indexed_pkgs
232     print('Packages not indexed = %i out of %i' % (len(pkgs_not_indexed),
233                                                    len(pkgs)))
234     for pkg_id in pkgs_not_indexed:
235         pkg = model.Session.query(model.Package).get(pkg_id)
236         print((pkg.metadata_modified.strftime('%Y-%m-%d'), pkg.name))
237 
238 
239 def show(package_reference):
240     package_query = query_for(model.Package)
241     return package_query.get_index(package_reference)
242 
243 
244 def clear(package_reference):
245     package_index = index_for(model.Package)
246     log.debug("Clearing search index for dataset %s..." %
247               package_reference)
248     package_index.delete_package({'id': package_reference})
249 
250 
251 def clear_all():
252     package_index = index_for(model.Package)
253     log.debug("Clearing search index...")
254     package_index.clear()
255 
256 def _get_schema_from_solr(file_offset):
257     solr_url, solr_user, solr_password = SolrSettings.get()
258 
259     http_auth = None
260     if solr_user is not None and solr_password is not None:
261         http_auth = solr_user + ':' + solr_password
262         http_auth = 'Basic ' + http_auth.encode('base64').strip()
263 
264     url = solr_url.strip('/') + file_offset
265 
266     if http_auth:
267         response = requests.get(
268             url, headers={'Authorization': http_auth})
269     else:
270         response = requests.get(url)
271 
272     return response
273 
274 def check_solr_schema_version(schema_file=None):
275     '''
276         Checks if the schema version of the SOLR server is compatible
277         with this CKAN version.
278 
279         The schema will be retrieved from the SOLR server, using the
280         offset defined in SOLR_SCHEMA_FILE_OFFSET_MANAGED
281         ('/schema?wt=schema.xml'). If SOLR is set to use the manually
282         edited `schema.xml`, the schema will be retrieved from the SOLR
283         server using the offset defined in
284         SOLR_SCHEMA_FILE_OFFSET_CLASSIC ('/admin/file/?file=schema.xml').
285 
286         The schema_file parameter allows to override this pointing to
287         different schema file, but it should only be used for testing
288         purposes.
289 
290         If the CKAN instance is configured to not use SOLR or the SOLR
291         server is not available, the function will return False, as the
292         version check does not apply. If the SOLR server is available,
293         a SearchError exception will be thrown if the version could not
294         be extracted or it is not included in the supported versions list.
295 
296         :schema_file: Absolute path to an alternative schema file. Should
297                       be only used for testing purposes (Default is None)
298     '''
299 
300     if not is_available():
301         # Something is wrong with the SOLR server
302         log.warn('Problems were found while connecting to the SOLR server')
303         return False
304 
305     # Try to get the schema XML file to extract the version
306     if not schema_file:
307         try:
308             # Try Managed Schema
309             res = _get_schema_from_solr(SOLR_SCHEMA_FILE_OFFSET_MANAGED)
310             res.raise_for_status()
311         except requests.HTTPError:
312             # Fallback to Manually Edited schema.xml
313             res = _get_schema_from_solr(SOLR_SCHEMA_FILE_OFFSET_CLASSIC)
314         schema_content = res.text
315     else:
316         with open(schema_file, 'rb') as f:
317             schema_content = f.read()
318 
319     tree = xml.dom.minidom.parseString(schema_content)
320 
321     version = tree.documentElement.getAttribute('version')
322     if not len(version):
323         msg = 'Could not extract version info from the SOLR schema'
324         if schema_file:
325             msg += ', using file {}'.format(schema_file)
326         raise SearchError(msg)
327 
328     if not version in SUPPORTED_SCHEMA_VERSIONS:
329         raise SearchError('SOLR schema version not supported: %s. Supported'
330                           ' versions are [%s]'
331                           % (version, ', '.join(SUPPORTED_SCHEMA_VERSIONS)))
332     return True
333
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py
--- a/ckan/lib/search/__init__.py
+++ b/ckan/lib/search/__init__.py
@@ -5,6 +5,7 @@
 import sys
 import cgitb
 import warnings
+import base64
 import xml.dom.minidom
 
 import requests
@@ -259,7 +260,7 @@
     http_auth = None
     if solr_user is not None and solr_password is not None:
         http_auth = solr_user + ':' + solr_password
-        http_auth = 'Basic ' + http_auth.encode('base64').strip()
+        http_auth = 'Basic {}'.format(base64.b64encode(http_auth.encode('utf8')).strip())
 
     url = solr_url.strip('/') + file_offset

{"golden_diff": "diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py\n--- a/ckan/lib/search/__init__.py\n+++ b/ckan/lib/search/__init__.py\n@@ -5,6 +5,7 @@\n import sys\n import cgitb\n import warnings\n+import base64\n import xml.dom.minidom\n \n import requests\n@@ -259,7 +260,7 @@\n     http_auth = None\n     if solr_user is not None and solr_password is not None:\n         http_auth = solr_user + ':' + solr_password\n-        http_auth = 'Basic ' + http_auth.encode('base64').strip()\n+        http_auth = 'Basic {}'.format(base64.b64encode(http_auth.encode('utf8')).strip())\n \n     url = solr_url.strip('/') + file_offset\n", "issue": "Can't use `solr_user` and `solr_password` in Python 3\n**CKAN version**\r\n2.9 \r\n\r\n**Describe the bug**\r\nWhen setting the `solr_user` and `solr_password` config options running Python 3 you get a startup error:\r\n\r\n```\r\n  File \"/usr/lib/ckan/src/ckan/ckan/lib/search/__init__.py\", line 309, in check_solr_schema_version\r\n    res = _get_schema_from_solr(SOLR_SCHEMA_FILE_OFFSET_MANAGED)\r\n  File \"/usr/lib/ckan/src/ckan/ckan/lib/search/__init__.py\", line 262, in _get_schema_from_solr\r\n    http_auth = 'Basic ' + http_auth.encode('base64').strip()\r\nLookupError: 'base64' is not a text encoding; use codecs.encode() to handle arbitrary codecs\r\n```\r\n\n", "before_files": [{"content": "# encoding: utf-8\n\nfrom __future__ import print_function\nimport logging\nimport sys\nimport cgitb\nimport warnings\nimport xml.dom.minidom\n\nimport requests\n\nfrom ckan.common import asbool, config\nimport ckan.model as model\nimport ckan.plugins as p\nimport ckan.logic as logic\n\nfrom ckan.lib.search.common import (\n    SearchIndexError, SearchError, SearchQueryError,\n    make_connection, is_available, SolrSettings\n    )\nfrom ckan.lib.search.index import PackageSearchIndex, NoopSearchIndex\nfrom ckan.lib.search.query import (\n    TagSearchQuery, ResourceSearchQuery, PackageSearchQuery,\n    QueryOptions, convert_legacy_parameters_to_solr\n)\n\n\nlog = logging.getLogger(__name__)\n\n\ndef text_traceback():\n    with warnings.catch_warnings():\n        warnings.simplefilter(\"ignore\")\n        res = 'the original traceback:'.join(\n            cgitb.text(sys.exc_info()).split('the original traceback:')[1:]\n        ).strip()\n    return res\n\n\nSUPPORTED_SCHEMA_VERSIONS = ['2.8', '2.9']\n\nDEFAULT_OPTIONS = {\n    'limit': 20,\n    'offset': 0,\n    # about presenting the results\n    'order_by': 'rank',\n    'return_objects': False,\n    'ref_entity_with_attr': 'name',\n    'all_fields': False,\n    'search_tags': True,\n    'callback': None,  # simply passed through\n}\n\n_INDICES = {\n    'package': PackageSearchIndex\n}\n\n_QUERIES = {\n    'tag': TagSearchQuery,\n    'resource': ResourceSearchQuery,\n    'package': PackageSearchQuery\n}\n\nSOLR_SCHEMA_FILE_OFFSET_MANAGED = '/schema?wt=schema.xml'\nSOLR_SCHEMA_FILE_OFFSET_CLASSIC = '/admin/file/?file=schema.xml'\n\n\ndef _normalize_type(_type):\n    if isinstance(_type, model.domain_object.DomainObject):\n        _type = _type.__class__\n    if isinstance(_type, type):\n        _type = _type.__name__\n    return _type.strip().lower()\n\n\ndef index_for(_type):\n    \"\"\" Get a SearchIndex instance sub-class suitable for\n        the specified type. \"\"\"\n    try:\n        _type_n = _normalize_type(_type)\n        return _INDICES[_type_n]()\n    except KeyError as ke:\n        log.warn(\"Unknown search type: %s\" % _type)\n        return NoopSearchIndex()\n\n\ndef query_for(_type):\n    \"\"\" Get a SearchQuery instance sub-class suitable for the specified\n        type. \"\"\"\n    try:\n        _type_n = _normalize_type(_type)\n        return _QUERIES[_type_n]()\n    except KeyError as ke:\n        raise SearchError(\"Unknown search type: %s\" % _type)\n\n\ndef dispatch_by_operation(entity_type, entity, operation):\n    \"\"\"Call the appropriate index method for a given notification.\"\"\"\n    try:\n        index = index_for(entity_type)\n        if operation == model.domain_object.DomainObjectOperation.new:\n            index.insert_dict(entity)\n        elif operation == model.domain_object.DomainObjectOperation.changed:\n            index.update_dict(entity)\n        elif operation == model.domain_object.DomainObjectOperation.deleted:\n            index.remove_dict(entity)\n        else:\n            log.warn(\"Unknown operation: %s\" % operation)\n    except Exception as ex:\n        log.exception(ex)\n        # we really need to know about any exceptions, so reraise\n        # (see #1172)\n        raise\n\n\nclass SynchronousSearchPlugin(p.SingletonPlugin):\n    \"\"\"Update the search index automatically.\"\"\"\n    p.implements(p.IDomainObjectModification, inherit=True)\n\n    def notify(self, entity, operation):\n        if (not isinstance(entity, model.Package) or\n                not asbool(config.get('ckan.search.automatic_indexing', True))):\n            return\n        if operation != model.domain_object.DomainObjectOperation.deleted:\n            dispatch_by_operation(\n                entity.__class__.__name__,\n                logic.get_action('package_show')(\n                    {'model': model, 'ignore_auth': True, 'validate': False,\n                     'use_cache': False},\n                    {'id': entity.id}),\n                operation\n            )\n        elif operation == model.domain_object.DomainObjectOperation.deleted:\n            dispatch_by_operation(entity.__class__.__name__,\n                                  {'id': entity.id}, operation)\n        else:\n            log.warn(\"Discarded Sync. indexing for: %s\" % entity)\n\n\ndef rebuild(package_id=None, only_missing=False, force=False, refresh=False,\n            defer_commit=False, package_ids=None, quiet=False):\n    '''\n        Rebuilds the search index.\n\n        If a dataset id is provided, only this dataset will be reindexed.\n        When reindexing all datasets, if only_missing is True, only the\n        datasets not already indexed will be processed. If force equals\n        True, if an exception is found, the exception will be logged, but\n        the process will carry on.\n    '''\n    log.info(\"Rebuilding search index...\")\n\n    package_index = index_for(model.Package)\n    context = {'model': model, 'ignore_auth': True, 'validate': False,\n        'use_cache': False}\n\n    if package_id:\n        pkg_dict = logic.get_action('package_show')(context,\n            {'id': package_id})\n        log.info('Indexing just package %r...', pkg_dict['name'])\n        package_index.remove_dict(pkg_dict)\n        package_index.insert_dict(pkg_dict)\n    elif package_ids is not None:\n        for package_id in package_ids:\n            pkg_dict = logic.get_action('package_show')(context,\n                {'id': package_id})\n            log.info('Indexing just package %r...', pkg_dict['name'])\n            package_index.update_dict(pkg_dict, True)\n    else:\n        package_ids = [r[0] for r in model.Session.query(model.Package.id).\n                       filter(model.Package.state != 'deleted').all()]\n        if only_missing:\n            log.info('Indexing only missing packages...')\n            package_query = query_for(model.Package)\n            indexed_pkg_ids = set(package_query.get_all_entity_ids(\n                max_results=len(package_ids)))\n            # Packages not indexed\n            package_ids = set(package_ids) - indexed_pkg_ids\n\n            if len(package_ids) == 0:\n                log.info('All datasets are already indexed')\n                return\n        else:\n            log.info('Rebuilding the whole index...')\n            # When refreshing, the index is not previously cleared\n            if not refresh:\n                package_index.clear()\n\n        total_packages = len(package_ids)\n        for counter, pkg_id in enumerate(package_ids):\n            if not quiet:\n                sys.stdout.write(\n                    \"\\rIndexing dataset {0}/{1}\".format(\n                        counter +1, total_packages)\n                )\n                sys.stdout.flush()\n            try:\n                package_index.update_dict(\n                    logic.get_action('package_show')(context,\n                        {'id': pkg_id}\n                    ),\n                    defer_commit\n                )\n            except Exception as e:\n                log.error(u'Error while indexing dataset %s: %s' %\n                          (pkg_id, repr(e)))\n                if force:\n                    log.error(text_traceback())\n                    continue\n                else:\n                    raise\n\n    model.Session.commit()\n    log.info('Finished rebuilding search index.')\n\n\ndef commit():\n    package_index = index_for(model.Package)\n    package_index.commit()\n    log.info('Commited pending changes on the search index')\n\n\ndef check():\n    package_query = query_for(model.Package)\n\n    log.debug(\"Checking packages search index...\")\n    pkgs_q = model.Session.query(model.Package).filter_by(\n        state=model.State.ACTIVE)\n    pkgs = {pkg.id for pkg in pkgs_q}\n    indexed_pkgs = set(package_query.get_all_entity_ids(max_results=len(pkgs)))\n    pkgs_not_indexed = pkgs - indexed_pkgs\n    print('Packages not indexed = %i out of %i' % (len(pkgs_not_indexed),\n                                                   len(pkgs)))\n    for pkg_id in pkgs_not_indexed:\n        pkg = model.Session.query(model.Package).get(pkg_id)\n        print((pkg.metadata_modified.strftime('%Y-%m-%d'), pkg.name))\n\n\ndef show(package_reference):\n    package_query = query_for(model.Package)\n    return package_query.get_index(package_reference)\n\n\ndef clear(package_reference):\n    package_index = index_for(model.Package)\n    log.debug(\"Clearing search index for dataset %s...\" %\n              package_reference)\n    package_index.delete_package({'id': package_reference})\n\n\ndef clear_all():\n    package_index = index_for(model.Package)\n    log.debug(\"Clearing search index...\")\n    package_index.clear()\n\ndef _get_schema_from_solr(file_offset):\n    solr_url, solr_user, solr_password = SolrSettings.get()\n\n    http_auth = None\n    if solr_user is not None and solr_password is not None:\n        http_auth = solr_user + ':' + solr_password\n        http_auth = 'Basic ' + http_auth.encode('base64').strip()\n\n    url = solr_url.strip('/') + file_offset\n\n    if http_auth:\n        response = requests.get(\n            url, headers={'Authorization': http_auth})\n    else:\n        response = requests.get(url)\n\n    return response\n\ndef check_solr_schema_version(schema_file=None):\n    '''\n        Checks if the schema version of the SOLR server is compatible\n        with this CKAN version.\n\n        The schema will be retrieved from the SOLR server, using the\n        offset defined in SOLR_SCHEMA_FILE_OFFSET_MANAGED\n        ('/schema?wt=schema.xml'). If SOLR is set to use the manually\n        edited `schema.xml`, the schema will be retrieved from the SOLR\n        server using the offset defined in\n        SOLR_SCHEMA_FILE_OFFSET_CLASSIC ('/admin/file/?file=schema.xml').\n\n        The schema_file parameter allows to override this pointing to\n        different schema file, but it should only be used for testing\n        purposes.\n\n        If the CKAN instance is configured to not use SOLR or the SOLR\n        server is not available, the function will return False, as the\n        version check does not apply. If the SOLR server is available,\n        a SearchError exception will be thrown if the version could not\n        be extracted or it is not included in the supported versions list.\n\n        :schema_file: Absolute path to an alternative schema file. Should\n                      be only used for testing purposes (Default is None)\n    '''\n\n    if not is_available():\n        # Something is wrong with the SOLR server\n        log.warn('Problems were found while connecting to the SOLR server')\n        return False\n\n    # Try to get the schema XML file to extract the version\n    if not schema_file:\n        try:\n            # Try Managed Schema\n            res = _get_schema_from_solr(SOLR_SCHEMA_FILE_OFFSET_MANAGED)\n            res.raise_for_status()\n        except requests.HTTPError:\n            # Fallback to Manually Edited schema.xml\n            res = _get_schema_from_solr(SOLR_SCHEMA_FILE_OFFSET_CLASSIC)\n        schema_content = res.text\n    else:\n        with open(schema_file, 'rb') as f:\n            schema_content = f.read()\n\n    tree = xml.dom.minidom.parseString(schema_content)\n\n    version = tree.documentElement.getAttribute('version')\n    if not len(version):\n        msg = 'Could not extract version info from the SOLR schema'\n        if schema_file:\n            msg += ', using file {}'.format(schema_file)\n        raise SearchError(msg)\n\n    if not version in SUPPORTED_SCHEMA_VERSIONS:\n        raise SearchError('SOLR schema version not supported: %s. Supported'\n                          ' versions are [%s]'\n                          % (version, ', '.join(SUPPORTED_SCHEMA_VERSIONS)))\n    return True\n", "path": "ckan/lib/search/__init__.py"}], "after_files": [{"content": "# encoding: utf-8\n\nfrom __future__ import print_function\nimport logging\nimport sys\nimport cgitb\nimport warnings\nimport base64\nimport xml.dom.minidom\n\nimport requests\n\nfrom ckan.common import asbool, config\nimport ckan.model as model\nimport ckan.plugins as p\nimport ckan.logic as logic\n\nfrom ckan.lib.search.common import (\n    SearchIndexError, SearchError, SearchQueryError,\n    make_connection, is_available, SolrSettings\n    )\nfrom ckan.lib.search.index import PackageSearchIndex, NoopSearchIndex\nfrom ckan.lib.search.query import (\n    TagSearchQuery, ResourceSearchQuery, PackageSearchQuery,\n    QueryOptions, convert_legacy_parameters_to_solr\n)\n\n\nlog = logging.getLogger(__name__)\n\n\ndef text_traceback():\n    with warnings.catch_warnings():\n        warnings.simplefilter(\"ignore\")\n        res = 'the original traceback:'.join(\n            cgitb.text(sys.exc_info()).split('the original traceback:')[1:]\n        ).strip()\n    return res\n\n\nSUPPORTED_SCHEMA_VERSIONS = ['2.8', '2.9']\n\nDEFAULT_OPTIONS = {\n    'limit': 20,\n    'offset': 0,\n    # about presenting the results\n    'order_by': 'rank',\n    'return_objects': False,\n    'ref_entity_with_attr': 'name',\n    'all_fields': False,\n    'search_tags': True,\n    'callback': None,  # simply passed through\n}\n\n_INDICES = {\n    'package': PackageSearchIndex\n}\n\n_QUERIES = {\n    'tag': TagSearchQuery,\n    'resource': ResourceSearchQuery,\n    'package': PackageSearchQuery\n}\n\nSOLR_SCHEMA_FILE_OFFSET_MANAGED = '/schema?wt=schema.xml'\nSOLR_SCHEMA_FILE_OFFSET_CLASSIC = '/admin/file/?file=schema.xml'\n\n\ndef _normalize_type(_type):\n    if isinstance(_type, model.domain_object.DomainObject):\n        _type = _type.__class__\n    if isinstance(_type, type):\n        _type = _type.__name__\n    return _type.strip().lower()\n\n\ndef index_for(_type):\n    \"\"\" Get a SearchIndex instance sub-class suitable for\n        the specified type. \"\"\"\n    try:\n        _type_n = _normalize_type(_type)\n        return _INDICES[_type_n]()\n    except KeyError as ke:\n        log.warn(\"Unknown search type: %s\" % _type)\n        return NoopSearchIndex()\n\n\ndef query_for(_type):\n    \"\"\" Get a SearchQuery instance sub-class suitable for the specified\n        type. \"\"\"\n    try:\n        _type_n = _normalize_type(_type)\n        return _QUERIES[_type_n]()\n    except KeyError as ke:\n        raise SearchError(\"Unknown search type: %s\" % _type)\n\n\ndef dispatch_by_operation(entity_type, entity, operation):\n    \"\"\"Call the appropriate index method for a given notification.\"\"\"\n    try:\n        index = index_for(entity_type)\n        if operation == model.domain_object.DomainObjectOperation.new:\n            index.insert_dict(entity)\n        elif operation == model.domain_object.DomainObjectOperation.changed:\n            index.update_dict(entity)\n        elif operation == model.domain_object.DomainObjectOperation.deleted:\n            index.remove_dict(entity)\n        else:\n            log.warn(\"Unknown operation: %s\" % operation)\n    except Exception as ex:\n        log.exception(ex)\n        # we really need to know about any exceptions, so reraise\n        # (see #1172)\n        raise\n\n\nclass SynchronousSearchPlugin(p.SingletonPlugin):\n    \"\"\"Update the search index automatically.\"\"\"\n    p.implements(p.IDomainObjectModification, inherit=True)\n\n    def notify(self, entity, operation):\n        if (not isinstance(entity, model.Package) or\n                not asbool(config.get('ckan.search.automatic_indexing', True))):\n            return\n        if operation != model.domain_object.DomainObjectOperation.deleted:\n            dispatch_by_operation(\n                entity.__class__.__name__,\n                logic.get_action('package_show')(\n                    {'model': model, 'ignore_auth': True, 'validate': False,\n                     'use_cache': False},\n                    {'id': entity.id}),\n                operation\n            )\n        elif operation == model.domain_object.DomainObjectOperation.deleted:\n            dispatch_by_operation(entity.__class__.__name__,\n                                  {'id': entity.id}, operation)\n        else:\n            log.warn(\"Discarded Sync. indexing for: %s\" % entity)\n\n\ndef rebuild(package_id=None, only_missing=False, force=False, refresh=False,\n            defer_commit=False, package_ids=None, quiet=False):\n    '''\n        Rebuilds the search index.\n\n        If a dataset id is provided, only this dataset will be reindexed.\n        When reindexing all datasets, if only_missing is True, only the\n        datasets not already indexed will be processed. If force equals\n        True, if an exception is found, the exception will be logged, but\n        the process will carry on.\n    '''\n    log.info(\"Rebuilding search index...\")\n\n    package_index = index_for(model.Package)\n    context = {'model': model, 'ignore_auth': True, 'validate': False,\n        'use_cache': False}\n\n    if package_id:\n        pkg_dict = logic.get_action('package_show')(context,\n            {'id': package_id})\n        log.info('Indexing just package %r...', pkg_dict['name'])\n        package_index.remove_dict(pkg_dict)\n        package_index.insert_dict(pkg_dict)\n    elif package_ids is not None:\n        for package_id in package_ids:\n            pkg_dict = logic.get_action('package_show')(context,\n                {'id': package_id})\n            log.info('Indexing just package %r...', pkg_dict['name'])\n            package_index.update_dict(pkg_dict, True)\n    else:\n        package_ids = [r[0] for r in model.Session.query(model.Package.id).\n                       filter(model.Package.state != 'deleted').all()]\n        if only_missing:\n            log.info('Indexing only missing packages...')\n            package_query = query_for(model.Package)\n            indexed_pkg_ids = set(package_query.get_all_entity_ids(\n                max_results=len(package_ids)))\n            # Packages not indexed\n            package_ids = set(package_ids) - indexed_pkg_ids\n\n            if len(package_ids) == 0:\n                log.info('All datasets are already indexed')\n                return\n        else:\n            log.info('Rebuilding the whole index...')\n            # When refreshing, the index is not previously cleared\n            if not refresh:\n                package_index.clear()\n\n        total_packages = len(package_ids)\n        for counter, pkg_id in enumerate(package_ids):\n            if not quiet:\n                sys.stdout.write(\n                    \"\\rIndexing dataset {0}/{1}\".format(\n                        counter +1, total_packages)\n                )\n                sys.stdout.flush()\n            try:\n                package_index.update_dict(\n                    logic.get_action('package_show')(context,\n                        {'id': pkg_id}\n                    ),\n                    defer_commit\n                )\n            except Exception as e:\n                log.error(u'Error while indexing dataset %s: %s' %\n                          (pkg_id, repr(e)))\n                if force:\n                    log.error(text_traceback())\n                    continue\n                else:\n                    raise\n\n    model.Session.commit()\n    log.info('Finished rebuilding search index.')\n\n\ndef commit():\n    package_index = index_for(model.Package)\n    package_index.commit()\n    log.info('Commited pending changes on the search index')\n\n\ndef check():\n    package_query = query_for(model.Package)\n\n    log.debug(\"Checking packages search index...\")\n    pkgs_q = model.Session.query(model.Package).filter_by(\n        state=model.State.ACTIVE)\n    pkgs = {pkg.id for pkg in pkgs_q}\n    indexed_pkgs = set(package_query.get_all_entity_ids(max_results=len(pkgs)))\n    pkgs_not_indexed = pkgs - indexed_pkgs\n    print('Packages not indexed = %i out of %i' % (len(pkgs_not_indexed),\n                                                   len(pkgs)))\n    for pkg_id in pkgs_not_indexed:\n        pkg = model.Session.query(model.Package).get(pkg_id)\n        print((pkg.metadata_modified.strftime('%Y-%m-%d'), pkg.name))\n\n\ndef show(package_reference):\n    package_query = query_for(model.Package)\n    return package_query.get_index(package_reference)\n\n\ndef clear(package_reference):\n    package_index = index_for(model.Package)\n    log.debug(\"Clearing search index for dataset %s...\" %\n              package_reference)\n    package_index.delete_package({'id': package_reference})\n\n\ndef clear_all():\n    package_index = index_for(model.Package)\n    log.debug(\"Clearing search index...\")\n    package_index.clear()\n\ndef _get_schema_from_solr(file_offset):\n    solr_url, solr_user, solr_password = SolrSettings.get()\n\n    http_auth = None\n    if solr_user is not None and solr_password is not None:\n        http_auth = solr_user + ':' + solr_password\n        http_auth = 'Basic {}'.format(base64.b64encode(http_auth.encode('utf8')).strip())\n\n    url = solr_url.strip('/') + file_offset\n\n    if http_auth:\n        response = requests.get(\n            url, headers={'Authorization': http_auth})\n    else:\n        response = requests.get(url)\n\n    return response\n\ndef check_solr_schema_version(schema_file=None):\n    '''\n        Checks if the schema version of the SOLR server is compatible\n        with this CKAN version.\n\n        The schema will be retrieved from the SOLR server, using the\n        offset defined in SOLR_SCHEMA_FILE_OFFSET_MANAGED\n        ('/schema?wt=schema.xml'). If SOLR is set to use the manually\n        edited `schema.xml`, the schema will be retrieved from the SOLR\n        server using the offset defined in\n        SOLR_SCHEMA_FILE_OFFSET_CLASSIC ('/admin/file/?file=schema.xml').\n\n        The schema_file parameter allows to override this pointing to\n        different schema file, but it should only be used for testing\n        purposes.\n\n        If the CKAN instance is configured to not use SOLR or the SOLR\n        server is not available, the function will return False, as the\n        version check does not apply. If the SOLR server is available,\n        a SearchError exception will be thrown if the version could not\n        be extracted or it is not included in the supported versions list.\n\n        :schema_file: Absolute path to an alternative schema file. Should\n                      be only used for testing purposes (Default is None)\n    '''\n\n    if not is_available():\n        # Something is wrong with the SOLR server\n        log.warn('Problems were found while connecting to the SOLR server')\n        return False\n\n    # Try to get the schema XML file to extract the version\n    if not schema_file:\n        try:\n            # Try Managed Schema\n            res = _get_schema_from_solr(SOLR_SCHEMA_FILE_OFFSET_MANAGED)\n            res.raise_for_status()\n        except requests.HTTPError:\n            # Fallback to Manually Edited schema.xml\n            res = _get_schema_from_solr(SOLR_SCHEMA_FILE_OFFSET_CLASSIC)\n        schema_content = res.text\n    else:\n        with open(schema_file, 'rb') as f:\n            schema_content = f.read()\n\n    tree = xml.dom.minidom.parseString(schema_content)\n\n    version = tree.documentElement.getAttribute('version')\n    if not len(version):\n        msg = 'Could not extract version info from the SOLR schema'\n        if schema_file:\n            msg += ', using file {}'.format(schema_file)\n        raise SearchError(msg)\n\n    if not version in SUPPORTED_SCHEMA_VERSIONS:\n        raise SearchError('SOLR schema version not supported: %s. Supported'\n                          ' versions are [%s]'\n                          % (version, ', '.join(SUPPORTED_SCHEMA_VERSIONS)))\n    return True\n", "path": "ckan/lib/search/__init__.py"}]}