We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Flick engine is broken
The flick engine seems to be broken : no result 

In the console :

```
INFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): secure.flickr.com
DEBUG:requests.packages.urllib3.connectionpool:"GET /search/?text=test&page=1 HTTP/1.1" 302 20
DEBUG:searx.search:flickr redirect on: <Response [302]>
DEBUG:requests.packages.urllib3.connectionpool:"GET /search/?rb=1&text=test&page=1 HTTP/1.1" 302 91
DEBUG:searx.search:flickr redirect on: <Response [302]>
DEBUG:requests.packages.urllib3.connectionpool:"GET /browser/upgrade/?continue=/search/?rb=1&text=test&page=1 HTTP/1.1" 200 None
```


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `searx/engines/flickr_noapi.py`
Content:
```
1 #!/usr/bin/env python
2 
3 #  Flickr (Images)
4 #
5 # @website     https://www.flickr.com
6 # @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)
7 #
8 # @using-api   no
9 # @results     HTML
10 # @stable      no
11 # @parse       url, title, thumbnail, img_src
12 
13 from urllib import urlencode
14 from json import loads
15 import re
16 from searx.engines import logger
17 
18 
19 logger = logger.getChild('flickr-noapi')
20 
21 categories = ['images']
22 
23 url = 'https://secure.flickr.com/'
24 search_url = url + 'search/?{query}&page={page}'
25 photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
26 regex = re.compile(r"\"search-photos-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)
27 image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
28 
29 paging = True
30 
31 
32 def build_flickr_url(user_id, photo_id):
33     return photo_url.format(userid=user_id, photoid=photo_id)
34 
35 
36 def request(query, params):
37     params['url'] = search_url.format(query=urlencode({'text': query}),
38                                       page=params['pageno'])
39     return params
40 
41 
42 def response(resp):
43     results = []
44 
45     matches = regex.search(resp.text)
46 
47     if matches is None:
48         return results
49 
50     match = matches.group(1)
51     search_results = loads(match)
52 
53     if '_data' not in search_results:
54         return []
55 
56     photos = search_results['_data']
57 
58     for photo in photos:
59 
60         # In paged configuration, the first pages' photos
61         # are represented by a None object
62         if photo is None:
63             continue
64 
65         img_src = None
66         # From the biggest to the lowest format
67         for image_size in image_sizes:
68             if image_size in photo['sizes']:
69                 img_src = photo['sizes'][image_size]['url']
70                 break
71 
72         if not img_src:
73             logger.debug('cannot find valid image size: {0}'.format(repr(photo)))
74             continue
75 
76         if 'id' not in photo['owner']:
77             continue
78 
79 # For a bigger thumbnail, keep only the url_z, not the url_n
80         if 'n' in photo['sizes']:
81             thumbnail_src = photo['sizes']['n']['url']
82         elif 'z' in photo['sizes']:
83             thumbnail_src = photo['sizes']['z']['url']
84         else:
85             thumbnail_src = img_src
86 
87         url = build_flickr_url(photo['owner']['id'], photo['id'])
88 
89         title = photo.get('title', '')
90 
91         content = '<span class="photo-author">' +\
92                   photo['owner']['username'] +\
93                   '</span><br />'
94 
95         if 'description' in photo:
96             content = content +\
97                 '<span class="description">' +\
98                 photo['description'] +\
99                 '</span>'
100 
101         # append result
102         results.append({'url': url,
103                         'title': title,
104                         'img_src': img_src,
105                         'thumbnail_src': thumbnail_src,
106                         'content': content,
107                         'template': 'images.html'})
108 
109     return results
110
```

Path: `searx/utils.py`
Content:
```
1 # import htmlentitydefs
2 import locale
3 import dateutil.parser
4 import cStringIO
5 import csv
6 import os
7 import re
8 
9 from codecs import getincrementalencoder
10 from HTMLParser import HTMLParser
11 from random import choice
12 
13 from searx.version import VERSION_STRING
14 from searx import settings
15 from searx import logger
16 
17 
18 logger = logger.getChild('utils')
19 
20 ua_versions = ('31.0',
21                '32.0',
22                '33.0',
23                '34.0',
24                '35.0')
25 
26 ua_os = ('Windows NT 6.3; WOW64',
27          'X11; Linux x86_64',
28          'X11; Linux x86')
29 
30 ua = "Mozilla/5.0 ({os}) Gecko/20100101 Firefox/{version}"
31 
32 blocked_tags = ('script',
33                 'style')
34 
35 
36 def gen_useragent():
37     # TODO
38     return ua.format(os=choice(ua_os), version=choice(ua_versions))
39 
40 
41 def searx_useragent():
42     return 'searx/{searx_version} {suffix}'.format(
43            searx_version=VERSION_STRING,
44            suffix=settings['server'].get('useragent_suffix', ''))
45 
46 
47 def highlight_content(content, query):
48 
49     if not content:
50         return None
51     # ignoring html contents
52     # TODO better html content detection
53     if content.find('<') != -1:
54         return content
55 
56     query = query.decode('utf-8')
57     if content.lower().find(query.lower()) > -1:
58         query_regex = u'({0})'.format(re.escape(query))
59         content = re.sub(query_regex, '<span class="highlight">\\1</span>',
60                          content, flags=re.I | re.U)
61     else:
62         regex_parts = []
63         for chunk in query.split():
64             if len(chunk) == 1:
65                 regex_parts.append(u'\W+{0}\W+'.format(re.escape(chunk)))
66             else:
67                 regex_parts.append(u'{0}'.format(re.escape(chunk)))
68         query_regex = u'({0})'.format('|'.join(regex_parts))
69         content = re.sub(query_regex, '<span class="highlight">\\1</span>',
70                          content, flags=re.I | re.U)
71 
72     return content
73 
74 
75 class HTMLTextExtractor(HTMLParser):
76     def __init__(self):
77         HTMLParser.__init__(self)
78         self.result = []
79         self.tags = []
80 
81     def handle_starttag(self, tag, attrs):
82         self.tags.append(tag)
83 
84     def handle_endtag(self, tag):
85         if not self.tags:
86             return
87 
88         if tag != self.tags[-1]:
89             raise Exception("invalid html")
90 
91         self.tags.pop()
92 
93     def is_valid_tag(self):
94         return not self.tags or self.tags[-1] not in blocked_tags
95 
96     def handle_data(self, d):
97         if not self.is_valid_tag():
98             return
99         self.result.append(d)
100 
101     def handle_charref(self, number):
102         if not self.is_valid_tag():
103             return
104         if number[0] in (u'x', u'X'):
105             codepoint = int(number[1:], 16)
106         else:
107             codepoint = int(number)
108         self.result.append(unichr(codepoint))
109 
110     def handle_entityref(self, name):
111         if not self.is_valid_tag():
112             return
113         # codepoint = htmlentitydefs.name2codepoint[name]
114         # self.result.append(unichr(codepoint))
115         self.result.append(name)
116 
117     def get_text(self):
118         return u''.join(self.result).strip()
119 
120 
121 def html_to_text(html):
122     html = html.replace('\n', ' ')
123     html = ' '.join(html.split())
124     s = HTMLTextExtractor()
125     s.feed(html)
126     return s.get_text()
127 
128 
129 class UnicodeWriter:
130     """
131     A CSV writer which will write rows to CSV file "f",
132     which is encoded in the given encoding.
133     """
134 
135     def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
136         # Redirect output to a queue
137         self.queue = cStringIO.StringIO()
138         self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
139         self.stream = f
140         self.encoder = getincrementalencoder(encoding)()
141 
142     def writerow(self, row):
143         unicode_row = []
144         for col in row:
145             if type(col) == str or type(col) == unicode:
146                 unicode_row.append(col.encode('utf-8').strip())
147             else:
148                 unicode_row.append(col)
149         self.writer.writerow(unicode_row)
150         # Fetch UTF-8 output from the queue ...
151         data = self.queue.getvalue()
152         data = data.decode("utf-8")
153         # ... and reencode it into the target encoding
154         data = self.encoder.encode(data)
155         # write to the target stream
156         self.stream.write(data)
157         # empty queue
158         self.queue.truncate(0)
159 
160     def writerows(self, rows):
161         for row in rows:
162             self.writerow(row)
163 
164 
165 def get_themes(root):
166     """Returns available themes list."""
167 
168     static_path = os.path.join(root, 'static')
169     templates_path = os.path.join(root, 'templates')
170 
171     themes = os.listdir(os.path.join(static_path, 'themes'))
172     return static_path, templates_path, themes
173 
174 
175 def get_static_files(base_path):
176     base_path = os.path.join(base_path, 'static')
177     static_files = set()
178     base_path_length = len(base_path) + 1
179     for directory, _, files in os.walk(base_path):
180         for filename in files:
181             f = os.path.join(directory[base_path_length:], filename)
182             static_files.add(f)
183     return static_files
184 
185 
186 def get_result_templates(base_path):
187     base_path = os.path.join(base_path, 'templates')
188     result_templates = set()
189     base_path_length = len(base_path) + 1
190     for directory, _, files in os.walk(base_path):
191         if directory.endswith('result_templates'):
192             for filename in files:
193                 f = os.path.join(directory[base_path_length:], filename)
194                 result_templates.add(f)
195     return result_templates
196 
197 
198 def format_date_by_locale(date_string, locale_string):
199     # strftime works only on dates after 1900
200     parsed_date = dateutil.parser.parse(date_string)
201     if parsed_date.year <= 1900:
202         return parsed_date.isoformat().split('T')[0]
203 
204     orig_locale = locale.getlocale()[0]
205     try:
206         locale.setlocale(locale.LC_ALL, locale_string)
207     except:
208         logger.warning('cannot set locale: {0}'.format(locale_string))
209     formatted_date = parsed_date.strftime(locale.nl_langinfo(locale.D_FMT))
210     try:
211         locale.setlocale(locale.LC_ALL, orig_locale)
212     except:
213         logger.warning('cannot set original locale: {0}'.format(orig_locale))
214     return formatted_date
215 
216 
217 def dict_subset(d, properties):
218     result = {}
219     for k in properties:
220         if k in d:
221             result[k] = d[k]
222     return result
223 
224 
225 def prettify_url(url):
226     if len(url) > 74:
227         return u'{0}[...]{1}'.format(url[:35], url[-35:])
228     else:
229         return url
230 
231 
232 def get_blocked_engines(engines, cookies):
233     if 'blocked_engines' not in cookies:
234         return [(engine_name, category) for engine_name in engines
235                 for category in engines[engine_name].categories if engines[engine_name].disabled]
236 
237     blocked_engine_strings = cookies.get('blocked_engines', '').split(',')
238     blocked_engines = []
239 
240     if not blocked_engine_strings:
241         return blocked_engines
242 
243     for engine_string in blocked_engine_strings:
244         if engine_string.find('__') > -1:
245             engine, category = engine_string.split('__', 1)
246             if engine in engines and category in engines[engine].categories:
247                 blocked_engines.append((engine, category))
248         elif engine_string in engines:
249             for category in engines[engine_string].categories:
250                 blocked_engines.append((engine_string, category))
251 
252     return blocked_engines
253
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py
--- a/searx/engines/flickr_noapi.py
+++ b/searx/engines/flickr_noapi.py
@@ -20,8 +20,8 @@
 
 categories = ['images']
 
-url = 'https://secure.flickr.com/'
-search_url = url + 'search/?{query}&page={page}'
+url = 'https://www.flickr.com/'
+search_url = url + 'search?{query}&page={page}'
 photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'
 regex = re.compile(r"\"search-photos-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)
 image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')
diff --git a/searx/utils.py b/searx/utils.py
--- a/searx/utils.py
+++ b/searx/utils.py
@@ -17,17 +17,16 @@
 
 logger = logger.getChild('utils')
 
-ua_versions = ('31.0',
-               '32.0',
-               '33.0',
+ua_versions = ('33.0',
                '34.0',
-               '35.0')
+               '35.0',
+               '36.0',
+               '37.0')
 
 ua_os = ('Windows NT 6.3; WOW64',
          'X11; Linux x86_64',
          'X11; Linux x86')
-
-ua = "Mozilla/5.0 ({os}) Gecko/20100101 Firefox/{version}"
+ua = "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}"
 
 blocked_tags = ('script',
                 'style')

{"golden_diff": "diff --git a/searx/engines/flickr_noapi.py b/searx/engines/flickr_noapi.py\n--- a/searx/engines/flickr_noapi.py\n+++ b/searx/engines/flickr_noapi.py\n@@ -20,8 +20,8 @@\n \n categories = ['images']\n \n-url = 'https://secure.flickr.com/'\n-search_url = url + 'search/?{query}&page={page}'\n+url = 'https://www.flickr.com/'\n+search_url = url + 'search?{query}&page={page}'\n photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'\n regex = re.compile(r\"\\\"search-photos-models\\\",\\\"photos\\\":(.*}),\\\"totalItems\\\":\", re.DOTALL)\n image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')\ndiff --git a/searx/utils.py b/searx/utils.py\n--- a/searx/utils.py\n+++ b/searx/utils.py\n@@ -17,17 +17,16 @@\n \n logger = logger.getChild('utils')\n \n-ua_versions = ('31.0',\n-               '32.0',\n-               '33.0',\n+ua_versions = ('33.0',\n                '34.0',\n-               '35.0')\n+               '35.0',\n+               '36.0',\n+               '37.0')\n \n ua_os = ('Windows NT 6.3; WOW64',\n          'X11; Linux x86_64',\n          'X11; Linux x86')\n-\n-ua = \"Mozilla/5.0 ({os}) Gecko/20100101 Firefox/{version}\"\n+ua = \"Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}\"\n \n blocked_tags = ('script',\n                 'style')\n", "issue": "Flick engine is broken\nThe flick engine seems to be broken : no result \n\nIn the console :\n\n```\nINFO:requests.packages.urllib3.connectionpool:Starting new HTTPS connection (1): secure.flickr.com\nDEBUG:requests.packages.urllib3.connectionpool:\"GET /search/?text=test&page=1 HTTP/1.1\" 302 20\nDEBUG:searx.search:flickr redirect on: <Response [302]>\nDEBUG:requests.packages.urllib3.connectionpool:\"GET /search/?rb=1&text=test&page=1 HTTP/1.1\" 302 91\nDEBUG:searx.search:flickr redirect on: <Response [302]>\nDEBUG:requests.packages.urllib3.connectionpool:\"GET /browser/upgrade/?continue=/search/?rb=1&text=test&page=1 HTTP/1.1\" 200 None\n```\n\n", "before_files": [{"content": "#!/usr/bin/env python\n\n#  Flickr (Images)\n#\n# @website     https://www.flickr.com\n# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)\n#\n# @using-api   no\n# @results     HTML\n# @stable      no\n# @parse       url, title, thumbnail, img_src\n\nfrom urllib import urlencode\nfrom json import loads\nimport re\nfrom searx.engines import logger\n\n\nlogger = logger.getChild('flickr-noapi')\n\ncategories = ['images']\n\nurl = 'https://secure.flickr.com/'\nsearch_url = url + 'search/?{query}&page={page}'\nphoto_url = 'https://www.flickr.com/photos/{userid}/{photoid}'\nregex = re.compile(r\"\\\"search-photos-models\\\",\\\"photos\\\":(.*}),\\\"totalItems\\\":\", re.DOTALL)\nimage_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')\n\npaging = True\n\n\ndef build_flickr_url(user_id, photo_id):\n    return photo_url.format(userid=user_id, photoid=photo_id)\n\n\ndef request(query, params):\n    params['url'] = search_url.format(query=urlencode({'text': query}),\n                                      page=params['pageno'])\n    return params\n\n\ndef response(resp):\n    results = []\n\n    matches = regex.search(resp.text)\n\n    if matches is None:\n        return results\n\n    match = matches.group(1)\n    search_results = loads(match)\n\n    if '_data' not in search_results:\n        return []\n\n    photos = search_results['_data']\n\n    for photo in photos:\n\n        # In paged configuration, the first pages' photos\n        # are represented by a None object\n        if photo is None:\n            continue\n\n        img_src = None\n        # From the biggest to the lowest format\n        for image_size in image_sizes:\n            if image_size in photo['sizes']:\n                img_src = photo['sizes'][image_size]['url']\n                break\n\n        if not img_src:\n            logger.debug('cannot find valid image size: {0}'.format(repr(photo)))\n            continue\n\n        if 'id' not in photo['owner']:\n            continue\n\n# For a bigger thumbnail, keep only the url_z, not the url_n\n        if 'n' in photo['sizes']:\n            thumbnail_src = photo['sizes']['n']['url']\n        elif 'z' in photo['sizes']:\n            thumbnail_src = photo['sizes']['z']['url']\n        else:\n            thumbnail_src = img_src\n\n        url = build_flickr_url(photo['owner']['id'], photo['id'])\n\n        title = photo.get('title', '')\n\n        content = '<span class=\"photo-author\">' +\\\n                  photo['owner']['username'] +\\\n                  '</span><br />'\n\n        if 'description' in photo:\n            content = content +\\\n                '<span class=\"description\">' +\\\n                photo['description'] +\\\n                '</span>'\n\n        # append result\n        results.append({'url': url,\n                        'title': title,\n                        'img_src': img_src,\n                        'thumbnail_src': thumbnail_src,\n                        'content': content,\n                        'template': 'images.html'})\n\n    return results\n", "path": "searx/engines/flickr_noapi.py"}, {"content": "# import htmlentitydefs\nimport locale\nimport dateutil.parser\nimport cStringIO\nimport csv\nimport os\nimport re\n\nfrom codecs import getincrementalencoder\nfrom HTMLParser import HTMLParser\nfrom random import choice\n\nfrom searx.version import VERSION_STRING\nfrom searx import settings\nfrom searx import logger\n\n\nlogger = logger.getChild('utils')\n\nua_versions = ('31.0',\n               '32.0',\n               '33.0',\n               '34.0',\n               '35.0')\n\nua_os = ('Windows NT 6.3; WOW64',\n         'X11; Linux x86_64',\n         'X11; Linux x86')\n\nua = \"Mozilla/5.0 ({os}) Gecko/20100101 Firefox/{version}\"\n\nblocked_tags = ('script',\n                'style')\n\n\ndef gen_useragent():\n    # TODO\n    return ua.format(os=choice(ua_os), version=choice(ua_versions))\n\n\ndef searx_useragent():\n    return 'searx/{searx_version} {suffix}'.format(\n           searx_version=VERSION_STRING,\n           suffix=settings['server'].get('useragent_suffix', ''))\n\n\ndef highlight_content(content, query):\n\n    if not content:\n        return None\n    # ignoring html contents\n    # TODO better html content detection\n    if content.find('<') != -1:\n        return content\n\n    query = query.decode('utf-8')\n    if content.lower().find(query.lower()) > -1:\n        query_regex = u'({0})'.format(re.escape(query))\n        content = re.sub(query_regex, '<span class=\"highlight\">\\\\1</span>',\n                         content, flags=re.I | re.U)\n    else:\n        regex_parts = []\n        for chunk in query.split():\n            if len(chunk) == 1:\n                regex_parts.append(u'\\W+{0}\\W+'.format(re.escape(chunk)))\n            else:\n                regex_parts.append(u'{0}'.format(re.escape(chunk)))\n        query_regex = u'({0})'.format('|'.join(regex_parts))\n        content = re.sub(query_regex, '<span class=\"highlight\">\\\\1</span>',\n                         content, flags=re.I | re.U)\n\n    return content\n\n\nclass HTMLTextExtractor(HTMLParser):\n    def __init__(self):\n        HTMLParser.__init__(self)\n        self.result = []\n        self.tags = []\n\n    def handle_starttag(self, tag, attrs):\n        self.tags.append(tag)\n\n    def handle_endtag(self, tag):\n        if not self.tags:\n            return\n\n        if tag != self.tags[-1]:\n            raise Exception(\"invalid html\")\n\n        self.tags.pop()\n\n    def is_valid_tag(self):\n        return not self.tags or self.tags[-1] not in blocked_tags\n\n    def handle_data(self, d):\n        if not self.is_valid_tag():\n            return\n        self.result.append(d)\n\n    def handle_charref(self, number):\n        if not self.is_valid_tag():\n            return\n        if number[0] in (u'x', u'X'):\n            codepoint = int(number[1:], 16)\n        else:\n            codepoint = int(number)\n        self.result.append(unichr(codepoint))\n\n    def handle_entityref(self, name):\n        if not self.is_valid_tag():\n            return\n        # codepoint = htmlentitydefs.name2codepoint[name]\n        # self.result.append(unichr(codepoint))\n        self.result.append(name)\n\n    def get_text(self):\n        return u''.join(self.result).strip()\n\n\ndef html_to_text(html):\n    html = html.replace('\\n', ' ')\n    html = ' '.join(html.split())\n    s = HTMLTextExtractor()\n    s.feed(html)\n    return s.get_text()\n\n\nclass UnicodeWriter:\n    \"\"\"\n    A CSV writer which will write rows to CSV file \"f\",\n    which is encoded in the given encoding.\n    \"\"\"\n\n    def __init__(self, f, dialect=csv.excel, encoding=\"utf-8\", **kwds):\n        # Redirect output to a queue\n        self.queue = cStringIO.StringIO()\n        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)\n        self.stream = f\n        self.encoder = getincrementalencoder(encoding)()\n\n    def writerow(self, row):\n        unicode_row = []\n        for col in row:\n            if type(col) == str or type(col) == unicode:\n                unicode_row.append(col.encode('utf-8').strip())\n            else:\n                unicode_row.append(col)\n        self.writer.writerow(unicode_row)\n        # Fetch UTF-8 output from the queue ...\n        data = self.queue.getvalue()\n        data = data.decode(\"utf-8\")\n        # ... and reencode it into the target encoding\n        data = self.encoder.encode(data)\n        # write to the target stream\n        self.stream.write(data)\n        # empty queue\n        self.queue.truncate(0)\n\n    def writerows(self, rows):\n        for row in rows:\n            self.writerow(row)\n\n\ndef get_themes(root):\n    \"\"\"Returns available themes list.\"\"\"\n\n    static_path = os.path.join(root, 'static')\n    templates_path = os.path.join(root, 'templates')\n\n    themes = os.listdir(os.path.join(static_path, 'themes'))\n    return static_path, templates_path, themes\n\n\ndef get_static_files(base_path):\n    base_path = os.path.join(base_path, 'static')\n    static_files = set()\n    base_path_length = len(base_path) + 1\n    for directory, _, files in os.walk(base_path):\n        for filename in files:\n            f = os.path.join(directory[base_path_length:], filename)\n            static_files.add(f)\n    return static_files\n\n\ndef get_result_templates(base_path):\n    base_path = os.path.join(base_path, 'templates')\n    result_templates = set()\n    base_path_length = len(base_path) + 1\n    for directory, _, files in os.walk(base_path):\n        if directory.endswith('result_templates'):\n            for filename in files:\n                f = os.path.join(directory[base_path_length:], filename)\n                result_templates.add(f)\n    return result_templates\n\n\ndef format_date_by_locale(date_string, locale_string):\n    # strftime works only on dates after 1900\n    parsed_date = dateutil.parser.parse(date_string)\n    if parsed_date.year <= 1900:\n        return parsed_date.isoformat().split('T')[0]\n\n    orig_locale = locale.getlocale()[0]\n    try:\n        locale.setlocale(locale.LC_ALL, locale_string)\n    except:\n        logger.warning('cannot set locale: {0}'.format(locale_string))\n    formatted_date = parsed_date.strftime(locale.nl_langinfo(locale.D_FMT))\n    try:\n        locale.setlocale(locale.LC_ALL, orig_locale)\n    except:\n        logger.warning('cannot set original locale: {0}'.format(orig_locale))\n    return formatted_date\n\n\ndef dict_subset(d, properties):\n    result = {}\n    for k in properties:\n        if k in d:\n            result[k] = d[k]\n    return result\n\n\ndef prettify_url(url):\n    if len(url) > 74:\n        return u'{0}[...]{1}'.format(url[:35], url[-35:])\n    else:\n        return url\n\n\ndef get_blocked_engines(engines, cookies):\n    if 'blocked_engines' not in cookies:\n        return [(engine_name, category) for engine_name in engines\n                for category in engines[engine_name].categories if engines[engine_name].disabled]\n\n    blocked_engine_strings = cookies.get('blocked_engines', '').split(',')\n    blocked_engines = []\n\n    if not blocked_engine_strings:\n        return blocked_engines\n\n    for engine_string in blocked_engine_strings:\n        if engine_string.find('__') > -1:\n            engine, category = engine_string.split('__', 1)\n            if engine in engines and category in engines[engine].categories:\n                blocked_engines.append((engine, category))\n        elif engine_string in engines:\n            for category in engines[engine_string].categories:\n                blocked_engines.append((engine_string, category))\n\n    return blocked_engines\n", "path": "searx/utils.py"}], "after_files": [{"content": "#!/usr/bin/env python\n\n#  Flickr (Images)\n#\n# @website     https://www.flickr.com\n# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)\n#\n# @using-api   no\n# @results     HTML\n# @stable      no\n# @parse       url, title, thumbnail, img_src\n\nfrom urllib import urlencode\nfrom json import loads\nimport re\nfrom searx.engines import logger\n\n\nlogger = logger.getChild('flickr-noapi')\n\ncategories = ['images']\n\nurl = 'https://www.flickr.com/'\nsearch_url = url + 'search?{query}&page={page}'\nphoto_url = 'https://www.flickr.com/photos/{userid}/{photoid}'\nregex = re.compile(r\"\\\"search-photos-models\\\",\\\"photos\\\":(.*}),\\\"totalItems\\\":\", re.DOTALL)\nimage_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')\n\npaging = True\n\n\ndef build_flickr_url(user_id, photo_id):\n    return photo_url.format(userid=user_id, photoid=photo_id)\n\n\ndef request(query, params):\n    params['url'] = search_url.format(query=urlencode({'text': query}),\n                                      page=params['pageno'])\n    return params\n\n\ndef response(resp):\n    results = []\n\n    matches = regex.search(resp.text)\n\n    if matches is None:\n        return results\n\n    match = matches.group(1)\n    search_results = loads(match)\n\n    if '_data' not in search_results:\n        return []\n\n    photos = search_results['_data']\n\n    for photo in photos:\n\n        # In paged configuration, the first pages' photos\n        # are represented by a None object\n        if photo is None:\n            continue\n\n        img_src = None\n        # From the biggest to the lowest format\n        for image_size in image_sizes:\n            if image_size in photo['sizes']:\n                img_src = photo['sizes'][image_size]['url']\n                break\n\n        if not img_src:\n            logger.debug('cannot find valid image size: {0}'.format(repr(photo)))\n            continue\n\n        if 'id' not in photo['owner']:\n            continue\n\n# For a bigger thumbnail, keep only the url_z, not the url_n\n        if 'n' in photo['sizes']:\n            thumbnail_src = photo['sizes']['n']['url']\n        elif 'z' in photo['sizes']:\n            thumbnail_src = photo['sizes']['z']['url']\n        else:\n            thumbnail_src = img_src\n\n        url = build_flickr_url(photo['owner']['id'], photo['id'])\n\n        title = photo.get('title', '')\n\n        content = '<span class=\"photo-author\">' +\\\n                  photo['owner']['username'] +\\\n                  '</span><br />'\n\n        if 'description' in photo:\n            content = content +\\\n                '<span class=\"description\">' +\\\n                photo['description'] +\\\n                '</span>'\n\n        # append result\n        results.append({'url': url,\n                        'title': title,\n                        'img_src': img_src,\n                        'thumbnail_src': thumbnail_src,\n                        'content': content,\n                        'template': 'images.html'})\n\n    return results\n", "path": "searx/engines/flickr_noapi.py"}, {"content": "# import htmlentitydefs\nimport locale\nimport dateutil.parser\nimport cStringIO\nimport csv\nimport os\nimport re\n\nfrom codecs import getincrementalencoder\nfrom HTMLParser import HTMLParser\nfrom random import choice\n\nfrom searx.version import VERSION_STRING\nfrom searx import settings\nfrom searx import logger\n\n\nlogger = logger.getChild('utils')\n\nua_versions = ('33.0',\n               '34.0',\n               '35.0',\n               '36.0',\n               '37.0')\n\nua_os = ('Windows NT 6.3; WOW64',\n         'X11; Linux x86_64',\n         'X11; Linux x86')\nua = \"Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}\"\n\nblocked_tags = ('script',\n                'style')\n\n\ndef gen_useragent():\n    # TODO\n    return ua.format(os=choice(ua_os), version=choice(ua_versions))\n\n\ndef searx_useragent():\n    return 'searx/{searx_version} {suffix}'.format(\n           searx_version=VERSION_STRING,\n           suffix=settings['server'].get('useragent_suffix', ''))\n\n\ndef highlight_content(content, query):\n\n    if not content:\n        return None\n    # ignoring html contents\n    # TODO better html content detection\n    if content.find('<') != -1:\n        return content\n\n    query = query.decode('utf-8')\n    if content.lower().find(query.lower()) > -1:\n        query_regex = u'({0})'.format(re.escape(query))\n        content = re.sub(query_regex, '<span class=\"highlight\">\\\\1</span>',\n                         content, flags=re.I | re.U)\n    else:\n        regex_parts = []\n        for chunk in query.split():\n            if len(chunk) == 1:\n                regex_parts.append(u'\\W+{0}\\W+'.format(re.escape(chunk)))\n            else:\n                regex_parts.append(u'{0}'.format(re.escape(chunk)))\n        query_regex = u'({0})'.format('|'.join(regex_parts))\n        content = re.sub(query_regex, '<span class=\"highlight\">\\\\1</span>',\n                         content, flags=re.I | re.U)\n\n    return content\n\n\nclass HTMLTextExtractor(HTMLParser):\n    def __init__(self):\n        HTMLParser.__init__(self)\n        self.result = []\n        self.tags = []\n\n    def handle_starttag(self, tag, attrs):\n        self.tags.append(tag)\n\n    def handle_endtag(self, tag):\n        if not self.tags:\n            return\n\n        if tag != self.tags[-1]:\n            raise Exception(\"invalid html\")\n\n        self.tags.pop()\n\n    def is_valid_tag(self):\n        return not self.tags or self.tags[-1] not in blocked_tags\n\n    def handle_data(self, d):\n        if not self.is_valid_tag():\n            return\n        self.result.append(d)\n\n    def handle_charref(self, number):\n        if not self.is_valid_tag():\n            return\n        if number[0] in (u'x', u'X'):\n            codepoint = int(number[1:], 16)\n        else:\n            codepoint = int(number)\n        self.result.append(unichr(codepoint))\n\n    def handle_entityref(self, name):\n        if not self.is_valid_tag():\n            return\n        # codepoint = htmlentitydefs.name2codepoint[name]\n        # self.result.append(unichr(codepoint))\n        self.result.append(name)\n\n    def get_text(self):\n        return u''.join(self.result).strip()\n\n\ndef html_to_text(html):\n    html = html.replace('\\n', ' ')\n    html = ' '.join(html.split())\n    s = HTMLTextExtractor()\n    s.feed(html)\n    return s.get_text()\n\n\nclass UnicodeWriter:\n    \"\"\"\n    A CSV writer which will write rows to CSV file \"f\",\n    which is encoded in the given encoding.\n    \"\"\"\n\n    def __init__(self, f, dialect=csv.excel, encoding=\"utf-8\", **kwds):\n        # Redirect output to a queue\n        self.queue = cStringIO.StringIO()\n        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)\n        self.stream = f\n        self.encoder = getincrementalencoder(encoding)()\n\n    def writerow(self, row):\n        unicode_row = []\n        for col in row:\n            if type(col) == str or type(col) == unicode:\n                unicode_row.append(col.encode('utf-8').strip())\n            else:\n                unicode_row.append(col)\n        self.writer.writerow(unicode_row)\n        # Fetch UTF-8 output from the queue ...\n        data = self.queue.getvalue()\n        data = data.decode(\"utf-8\")\n        # ... and reencode it into the target encoding\n        data = self.encoder.encode(data)\n        # write to the target stream\n        self.stream.write(data)\n        # empty queue\n        self.queue.truncate(0)\n\n    def writerows(self, rows):\n        for row in rows:\n            self.writerow(row)\n\n\ndef get_themes(root):\n    \"\"\"Returns available themes list.\"\"\"\n\n    static_path = os.path.join(root, 'static')\n    templates_path = os.path.join(root, 'templates')\n\n    themes = os.listdir(os.path.join(static_path, 'themes'))\n    return static_path, templates_path, themes\n\n\ndef get_static_files(base_path):\n    base_path = os.path.join(base_path, 'static')\n    static_files = set()\n    base_path_length = len(base_path) + 1\n    for directory, _, files in os.walk(base_path):\n        for filename in files:\n            f = os.path.join(directory[base_path_length:], filename)\n            static_files.add(f)\n    return static_files\n\n\ndef get_result_templates(base_path):\n    base_path = os.path.join(base_path, 'templates')\n    result_templates = set()\n    base_path_length = len(base_path) + 1\n    for directory, _, files in os.walk(base_path):\n        if directory.endswith('result_templates'):\n            for filename in files:\n                f = os.path.join(directory[base_path_length:], filename)\n                result_templates.add(f)\n    return result_templates\n\n\ndef format_date_by_locale(date_string, locale_string):\n    # strftime works only on dates after 1900\n    parsed_date = dateutil.parser.parse(date_string)\n    if parsed_date.year <= 1900:\n        return parsed_date.isoformat().split('T')[0]\n\n    orig_locale = locale.getlocale()[0]\n    try:\n        locale.setlocale(locale.LC_ALL, locale_string)\n    except:\n        logger.warning('cannot set locale: {0}'.format(locale_string))\n    formatted_date = parsed_date.strftime(locale.nl_langinfo(locale.D_FMT))\n    try:\n        locale.setlocale(locale.LC_ALL, orig_locale)\n    except:\n        logger.warning('cannot set original locale: {0}'.format(orig_locale))\n    return formatted_date\n\n\ndef dict_subset(d, properties):\n    result = {}\n    for k in properties:\n        if k in d:\n            result[k] = d[k]\n    return result\n\n\ndef prettify_url(url):\n    if len(url) > 74:\n        return u'{0}[...]{1}'.format(url[:35], url[-35:])\n    else:\n        return url\n\n\ndef get_blocked_engines(engines, cookies):\n    if 'blocked_engines' not in cookies:\n        return [(engine_name, category) for engine_name in engines\n                for category in engines[engine_name].categories if engines[engine_name].disabled]\n\n    blocked_engine_strings = cookies.get('blocked_engines', '').split(',')\n    blocked_engines = []\n\n    if not blocked_engine_strings:\n        return blocked_engines\n\n    for engine_string in blocked_engine_strings:\n        if engine_string.find('__') > -1:\n            engine, category = engine_string.split('__', 1)\n            if engine in engines and category in engines[engine].categories:\n                blocked_engines.append((engine, category))\n        elif engine_string in engines:\n            for category in engines[engine_string].categories:\n                blocked_engines.append((engine_string, category))\n\n    return blocked_engines\n", "path": "searx/utils.py"}]}