We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
url: UnicodeEncodeError
UnicodeEncodeError: 'ascii' codec can't encode character u'\u2014' in position 352:    | ordinal not in range(128) (file "/usr/local/lib/python2.7/dist-packages/sopel/modules/url.py", line | 191, in find_title)  

https://github.com/sopel-irc/sopel/blob/master/sopel/modules/url.py#L191


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `sopel/modules/url.py`
Content:
```
1 # coding=utf-8
2 """URL title module"""
3 # Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham
4 # Copyright 2012-2013 Elsie Powell
5 # Copyright 2013      Lior Ramati (firerogue517@gmail.com)
6 # Copyright © 2014 Elad Alfassa <elad@fedoraproject.org>
7 # Licensed under the Eiffel Forum License 2.
8 from __future__ import unicode_literals, absolute_import, print_function, division
9 
10 import re
11 from contextlib import closing
12 from sopel import web, tools
13 from sopel.module import commands, rule, example
14 from sopel.config.types import ValidatedAttribute, ListAttribute, StaticSection
15 
16 import requests
17 
18 url_finder = None
19 # These are used to clean up the title tag before actually parsing it. Not the
20 # world's best way to do this, but it'll do for now.
21 title_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)
22 quoted_title = re.compile('[\'"]<title>[\'"]', re.IGNORECASE)
23 # This is another regex that presumably does something important.
24 re_dcc = re.compile(r'(?i)dcc\ssend')
25 # This sets the maximum number of bytes that should be read in order to find
26 # the title. We don't want it too high, or a link to a big file/stream will
27 # just keep downloading until there's no more memory. 640k ought to be enough
28 # for anybody.
29 max_bytes = 655360
30 
31 
32 class UrlSection(StaticSection):
33     # TODO some validation rules maybe?
34     exclude = ListAttribute('exclude')
35     exclusion_char = ValidatedAttribute('exclusion_char', default='!')
36 
37 
38 def configure(config):
39     config.define_section('url', UrlSection)
40     config.url.configure_setting(
41         'exclude',
42         'Enter regular expressions for each URL you would like to exclude.'
43     )
44     config.url.configure_setting(
45         'exclusion_char',
46         'Enter a character which can be prefixed to suppress URL titling'
47     )
48 
49 
50 def setup(bot=None):
51     global url_finder
52 
53     # TODO figure out why this is needed, and get rid of it, because really?
54     if not bot:
55         return
56     bot.config.define_section('url', UrlSection)
57 
58     if bot.config.url.exclude:
59         regexes = [re.compile(s) for s in bot.config.url.exclude]
60     else:
61         regexes = []
62 
63     # We're keeping these in their own list, rather than putting then in the
64     # callbacks list because 1, it's easier to deal with modules that are still
65     # using this list, and not the newer callbacks list and 2, having a lambda
66     # just to pass is kinda ugly.
67     if not bot.memory.contains('url_exclude'):
68         bot.memory['url_exclude'] = regexes
69     else:
70         exclude = bot.memory['url_exclude']
71         if regexes:
72             exclude.extend(regexes)
73         bot.memory['url_exclude'] = exclude
74 
75     # Ensure that url_callbacks and last_seen_url are in memory
76     if not bot.memory.contains('url_callbacks'):
77         bot.memory['url_callbacks'] = tools.SopelMemory()
78     if not bot.memory.contains('last_seen_url'):
79         bot.memory['last_seen_url'] = tools.SopelMemory()
80 
81     url_finder = re.compile(r'(?u)(%s?(?:http|https|ftp)(?:://\S+))' %
82                             (bot.config.url.exclusion_char), re.IGNORECASE)
83 
84 
85 @commands('title')
86 @example('.title http://google.com', '[ Google ] - google.com')
87 def title_command(bot, trigger):
88     """
89     Show the title or URL information for the given URL, or the last URL seen
90     in this channel.
91     """
92     if not trigger.group(2):
93         if trigger.sender not in bot.memory['last_seen_url']:
94             return
95         matched = check_callbacks(bot, trigger,
96                                   bot.memory['last_seen_url'][trigger.sender],
97                                   True)
98         if matched:
99             return
100         else:
101             urls = [bot.memory['last_seen_url'][trigger.sender]]
102     else:
103         urls = re.findall(url_finder, trigger)
104 
105     results = process_urls(bot, trigger, urls)
106     for title, domain in results[:4]:
107         bot.reply('[ %s ] - %s' % (title, domain))
108 
109 
110 @rule('(?u).*(https?://\S+).*')
111 def title_auto(bot, trigger):
112     """
113     Automatically show titles for URLs. For shortened URLs/redirects, find
114     where the URL redirects to and show the title for that (or call a function
115     from another module to give more information).
116     """
117     if re.match(bot.config.core.prefix + 'title', trigger):
118         return
119 
120     # Avoid fetching known malicious links
121     if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:
122         if bot.memory['safety_cache'][trigger]['positives'] > 1:
123             return
124 
125     urls = re.findall(url_finder, trigger)
126     if len(urls) == 0:
127         return
128 
129     results = process_urls(bot, trigger, urls)
130     bot.memory['last_seen_url'][trigger.sender] = urls[-1]
131 
132     for title, domain in results[:4]:
133         message = '[ %s ] - %s' % (title, domain)
134         # Guard against responding to other instances of this bot.
135         if message != trigger:
136             bot.say(message)
137 
138 
139 def process_urls(bot, trigger, urls):
140     """
141     For each URL in the list, ensure that it isn't handled by another module.
142     If not, find where it redirects to, if anywhere. If that redirected URL
143     should be handled by another module, dispatch the callback for it.
144     Return a list of (title, hostname) tuples for each URL which is not handled by
145     another module.
146     """
147 
148     results = []
149     for url in urls:
150         if not url.startswith(bot.config.url.exclusion_char):
151             # Magic stuff to account for international domain names
152             try:
153                 url = web.iri_to_uri(url)
154             except:
155                 pass
156             # First, check that the URL we got doesn't match
157             matched = check_callbacks(bot, trigger, url, False)
158             if matched:
159                 continue
160             # Finally, actually show the URL
161             title = find_title(url)
162             if title:
163                 results.append((title, get_hostname(url)))
164     return results
165 
166 
167 def check_callbacks(bot, trigger, url, run=True):
168     """
169     Check the given URL against the callbacks list. If it matches, and ``run``
170     is given as ``True``, run the callback function, otherwise pass. Returns
171     ``True`` if the url matched anything in the callbacks list.
172     """
173     # Check if it matches the exclusion list first
174     matched = any(regex.search(url) for regex in bot.memory['url_exclude'])
175     # Then, check if there's anything in the callback list
176     for regex, function in tools.iteritems(bot.memory['url_callbacks']):
177         match = regex.search(url)
178         if match:
179             if run:
180                 function(bot, trigger, match)
181             matched = True
182     return matched
183 
184 
185 def find_title(url):
186     """Return the title for the given URL."""
187     response = requests.get(url, stream=True)
188     try:
189         content = ''
190         for byte in response.iter_content(chunk_size=512, decode_unicode=True):
191             content += str(byte)
192             if '</title>' in content or len(content) > max_bytes:
193                 break
194     except UnicodeDecodeError:
195         return  # Fail silently when data can't be decoded
196     finally:
197         # need to close the connexion because we have not read all the data
198         response.close()
199 
200     # Some cleanup that I don't really grok, but was in the original, so
201     # we'll keep it (with the compiled regexes made global) for now.
202     content = title_tag_data.sub(r'<\1title>', content)
203     content = quoted_title.sub('', content)
204 
205     start = content.find('<title>')
206     end = content.find('</title>')
207     if start == -1 or end == -1:
208         return
209     title = web.decode(content[start + 7:end])
210     title = title.strip()[:200]
211 
212     title = ' '.join(title.split())  # cleanly remove multiple spaces
213 
214     # More cryptic regex substitutions. This one looks to be myano's invention.
215     title = re_dcc.sub('', title)
216 
217     return title or None
218 
219 
220 def get_hostname(url):
221     idx = 7
222     if url.startswith('https://'):
223         idx = 8
224     elif url.startswith('ftp://'):
225         idx = 6
226     hostname = url[idx:]
227     slash = hostname.find('/')
228     if slash != -1:
229         hostname = hostname[:slash]
230     return hostname
231 
232 if __name__ == "__main__":
233     from sopel.test_tools import run_example_tests
234     run_example_tests(__file__)
235
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/sopel/modules/url.py b/sopel/modules/url.py
--- a/sopel/modules/url.py
+++ b/sopel/modules/url.py
@@ -188,7 +188,10 @@
     try:
         content = ''
         for byte in response.iter_content(chunk_size=512, decode_unicode=True):
-            content += str(byte)
+            if not isinstance(byte, bytes):
+                content += byte
+            else:
+                break
             if '</title>' in content or len(content) > max_bytes:
                 break
     except UnicodeDecodeError:

{"golden_diff": "diff --git a/sopel/modules/url.py b/sopel/modules/url.py\n--- a/sopel/modules/url.py\n+++ b/sopel/modules/url.py\n@@ -188,7 +188,10 @@\n     try:\n         content = ''\n         for byte in response.iter_content(chunk_size=512, decode_unicode=True):\n-            content += str(byte)\n+            if not isinstance(byte, bytes):\n+                content += byte\n+            else:\n+                break\n             if '</title>' in content or len(content) > max_bytes:\n                 break\n     except UnicodeDecodeError:\n", "issue": "url: UnicodeEncodeError\nUnicodeEncodeError: 'ascii' codec can't encode character u'\\u2014' in position 352:    | ordinal not in range(128) (file \"/usr/local/lib/python2.7/dist-packages/sopel/modules/url.py\", line | 191, in find_title)  \n\nhttps://github.com/sopel-irc/sopel/blob/master/sopel/modules/url.py#L191\n\n", "before_files": [{"content": "# coding=utf-8\n\"\"\"URL title module\"\"\"\n# Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham\n# Copyright 2012-2013 Elsie Powell\n# Copyright 2013      Lior Ramati (firerogue517@gmail.com)\n# Copyright \u00a9 2014 Elad Alfassa <elad@fedoraproject.org>\n# Licensed under the Eiffel Forum License 2.\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport re\nfrom contextlib import closing\nfrom sopel import web, tools\nfrom sopel.module import commands, rule, example\nfrom sopel.config.types import ValidatedAttribute, ListAttribute, StaticSection\n\nimport requests\n\nurl_finder = None\n# These are used to clean up the title tag before actually parsing it. Not the\n# world's best way to do this, but it'll do for now.\ntitle_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)\nquoted_title = re.compile('[\\'\"]<title>[\\'\"]', re.IGNORECASE)\n# This is another regex that presumably does something important.\nre_dcc = re.compile(r'(?i)dcc\\ssend')\n# This sets the maximum number of bytes that should be read in order to find\n# the title. We don't want it too high, or a link to a big file/stream will\n# just keep downloading until there's no more memory. 640k ought to be enough\n# for anybody.\nmax_bytes = 655360\n\n\nclass UrlSection(StaticSection):\n    # TODO some validation rules maybe?\n    exclude = ListAttribute('exclude')\n    exclusion_char = ValidatedAttribute('exclusion_char', default='!')\n\n\ndef configure(config):\n    config.define_section('url', UrlSection)\n    config.url.configure_setting(\n        'exclude',\n        'Enter regular expressions for each URL you would like to exclude.'\n    )\n    config.url.configure_setting(\n        'exclusion_char',\n        'Enter a character which can be prefixed to suppress URL titling'\n    )\n\n\ndef setup(bot=None):\n    global url_finder\n\n    # TODO figure out why this is needed, and get rid of it, because really?\n    if not bot:\n        return\n    bot.config.define_section('url', UrlSection)\n\n    if bot.config.url.exclude:\n        regexes = [re.compile(s) for s in bot.config.url.exclude]\n    else:\n        regexes = []\n\n    # We're keeping these in their own list, rather than putting then in the\n    # callbacks list because 1, it's easier to deal with modules that are still\n    # using this list, and not the newer callbacks list and 2, having a lambda\n    # just to pass is kinda ugly.\n    if not bot.memory.contains('url_exclude'):\n        bot.memory['url_exclude'] = regexes\n    else:\n        exclude = bot.memory['url_exclude']\n        if regexes:\n            exclude.extend(regexes)\n        bot.memory['url_exclude'] = exclude\n\n    # Ensure that url_callbacks and last_seen_url are in memory\n    if not bot.memory.contains('url_callbacks'):\n        bot.memory['url_callbacks'] = tools.SopelMemory()\n    if not bot.memory.contains('last_seen_url'):\n        bot.memory['last_seen_url'] = tools.SopelMemory()\n\n    url_finder = re.compile(r'(?u)(%s?(?:http|https|ftp)(?:://\\S+))' %\n                            (bot.config.url.exclusion_char), re.IGNORECASE)\n\n\n@commands('title')\n@example('.title http://google.com', '[ Google ] - google.com')\ndef title_command(bot, trigger):\n    \"\"\"\n    Show the title or URL information for the given URL, or the last URL seen\n    in this channel.\n    \"\"\"\n    if not trigger.group(2):\n        if trigger.sender not in bot.memory['last_seen_url']:\n            return\n        matched = check_callbacks(bot, trigger,\n                                  bot.memory['last_seen_url'][trigger.sender],\n                                  True)\n        if matched:\n            return\n        else:\n            urls = [bot.memory['last_seen_url'][trigger.sender]]\n    else:\n        urls = re.findall(url_finder, trigger)\n\n    results = process_urls(bot, trigger, urls)\n    for title, domain in results[:4]:\n        bot.reply('[ %s ] - %s' % (title, domain))\n\n\n@rule('(?u).*(https?://\\S+).*')\ndef title_auto(bot, trigger):\n    \"\"\"\n    Automatically show titles for URLs. For shortened URLs/redirects, find\n    where the URL redirects to and show the title for that (or call a function\n    from another module to give more information).\n    \"\"\"\n    if re.match(bot.config.core.prefix + 'title', trigger):\n        return\n\n    # Avoid fetching known malicious links\n    if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:\n        if bot.memory['safety_cache'][trigger]['positives'] > 1:\n            return\n\n    urls = re.findall(url_finder, trigger)\n    if len(urls) == 0:\n        return\n\n    results = process_urls(bot, trigger, urls)\n    bot.memory['last_seen_url'][trigger.sender] = urls[-1]\n\n    for title, domain in results[:4]:\n        message = '[ %s ] - %s' % (title, domain)\n        # Guard against responding to other instances of this bot.\n        if message != trigger:\n            bot.say(message)\n\n\ndef process_urls(bot, trigger, urls):\n    \"\"\"\n    For each URL in the list, ensure that it isn't handled by another module.\n    If not, find where it redirects to, if anywhere. If that redirected URL\n    should be handled by another module, dispatch the callback for it.\n    Return a list of (title, hostname) tuples for each URL which is not handled by\n    another module.\n    \"\"\"\n\n    results = []\n    for url in urls:\n        if not url.startswith(bot.config.url.exclusion_char):\n            # Magic stuff to account for international domain names\n            try:\n                url = web.iri_to_uri(url)\n            except:\n                pass\n            # First, check that the URL we got doesn't match\n            matched = check_callbacks(bot, trigger, url, False)\n            if matched:\n                continue\n            # Finally, actually show the URL\n            title = find_title(url)\n            if title:\n                results.append((title, get_hostname(url)))\n    return results\n\n\ndef check_callbacks(bot, trigger, url, run=True):\n    \"\"\"\n    Check the given URL against the callbacks list. If it matches, and ``run``\n    is given as ``True``, run the callback function, otherwise pass. Returns\n    ``True`` if the url matched anything in the callbacks list.\n    \"\"\"\n    # Check if it matches the exclusion list first\n    matched = any(regex.search(url) for regex in bot.memory['url_exclude'])\n    # Then, check if there's anything in the callback list\n    for regex, function in tools.iteritems(bot.memory['url_callbacks']):\n        match = regex.search(url)\n        if match:\n            if run:\n                function(bot, trigger, match)\n            matched = True\n    return matched\n\n\ndef find_title(url):\n    \"\"\"Return the title for the given URL.\"\"\"\n    response = requests.get(url, stream=True)\n    try:\n        content = ''\n        for byte in response.iter_content(chunk_size=512, decode_unicode=True):\n            content += str(byte)\n            if '</title>' in content or len(content) > max_bytes:\n                break\n    except UnicodeDecodeError:\n        return  # Fail silently when data can't be decoded\n    finally:\n        # need to close the connexion because we have not read all the data\n        response.close()\n\n    # Some cleanup that I don't really grok, but was in the original, so\n    # we'll keep it (with the compiled regexes made global) for now.\n    content = title_tag_data.sub(r'<\\1title>', content)\n    content = quoted_title.sub('', content)\n\n    start = content.find('<title>')\n    end = content.find('</title>')\n    if start == -1 or end == -1:\n        return\n    title = web.decode(content[start + 7:end])\n    title = title.strip()[:200]\n\n    title = ' '.join(title.split())  # cleanly remove multiple spaces\n\n    # More cryptic regex substitutions. This one looks to be myano's invention.\n    title = re_dcc.sub('', title)\n\n    return title or None\n\n\ndef get_hostname(url):\n    idx = 7\n    if url.startswith('https://'):\n        idx = 8\n    elif url.startswith('ftp://'):\n        idx = 6\n    hostname = url[idx:]\n    slash = hostname.find('/')\n    if slash != -1:\n        hostname = hostname[:slash]\n    return hostname\n\nif __name__ == \"__main__\":\n    from sopel.test_tools import run_example_tests\n    run_example_tests(__file__)\n", "path": "sopel/modules/url.py"}], "after_files": [{"content": "# coding=utf-8\n\"\"\"URL title module\"\"\"\n# Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham\n# Copyright 2012-2013 Elsie Powell\n# Copyright 2013      Lior Ramati (firerogue517@gmail.com)\n# Copyright \u00a9 2014 Elad Alfassa <elad@fedoraproject.org>\n# Licensed under the Eiffel Forum License 2.\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport re\nfrom contextlib import closing\nfrom sopel import web, tools\nfrom sopel.module import commands, rule, example\nfrom sopel.config.types import ValidatedAttribute, ListAttribute, StaticSection\n\nimport requests\n\nurl_finder = None\n# These are used to clean up the title tag before actually parsing it. Not the\n# world's best way to do this, but it'll do for now.\ntitle_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)\nquoted_title = re.compile('[\\'\"]<title>[\\'\"]', re.IGNORECASE)\n# This is another regex that presumably does something important.\nre_dcc = re.compile(r'(?i)dcc\\ssend')\n# This sets the maximum number of bytes that should be read in order to find\n# the title. We don't want it too high, or a link to a big file/stream will\n# just keep downloading until there's no more memory. 640k ought to be enough\n# for anybody.\nmax_bytes = 655360\n\n\nclass UrlSection(StaticSection):\n    # TODO some validation rules maybe?\n    exclude = ListAttribute('exclude')\n    exclusion_char = ValidatedAttribute('exclusion_char', default='!')\n\n\ndef configure(config):\n    config.define_section('url', UrlSection)\n    config.url.configure_setting(\n        'exclude',\n        'Enter regular expressions for each URL you would like to exclude.'\n    )\n    config.url.configure_setting(\n        'exclusion_char',\n        'Enter a character which can be prefixed to suppress URL titling'\n    )\n\n\ndef setup(bot=None):\n    global url_finder\n\n    # TODO figure out why this is needed, and get rid of it, because really?\n    if not bot:\n        return\n    bot.config.define_section('url', UrlSection)\n\n    if bot.config.url.exclude:\n        regexes = [re.compile(s) for s in bot.config.url.exclude]\n    else:\n        regexes = []\n\n    # We're keeping these in their own list, rather than putting then in the\n    # callbacks list because 1, it's easier to deal with modules that are still\n    # using this list, and not the newer callbacks list and 2, having a lambda\n    # just to pass is kinda ugly.\n    if not bot.memory.contains('url_exclude'):\n        bot.memory['url_exclude'] = regexes\n    else:\n        exclude = bot.memory['url_exclude']\n        if regexes:\n            exclude.extend(regexes)\n        bot.memory['url_exclude'] = exclude\n\n    # Ensure that url_callbacks and last_seen_url are in memory\n    if not bot.memory.contains('url_callbacks'):\n        bot.memory['url_callbacks'] = tools.SopelMemory()\n    if not bot.memory.contains('last_seen_url'):\n        bot.memory['last_seen_url'] = tools.SopelMemory()\n\n    url_finder = re.compile(r'(?u)(%s?(?:http|https|ftp)(?:://\\S+))' %\n                            (bot.config.url.exclusion_char), re.IGNORECASE)\n\n\n@commands('title')\n@example('.title http://google.com', '[ Google ] - google.com')\ndef title_command(bot, trigger):\n    \"\"\"\n    Show the title or URL information for the given URL, or the last URL seen\n    in this channel.\n    \"\"\"\n    if not trigger.group(2):\n        if trigger.sender not in bot.memory['last_seen_url']:\n            return\n        matched = check_callbacks(bot, trigger,\n                                  bot.memory['last_seen_url'][trigger.sender],\n                                  True)\n        if matched:\n            return\n        else:\n            urls = [bot.memory['last_seen_url'][trigger.sender]]\n    else:\n        urls = re.findall(url_finder, trigger)\n\n    results = process_urls(bot, trigger, urls)\n    for title, domain in results[:4]:\n        bot.reply('[ %s ] - %s' % (title, domain))\n\n\n@rule('(?u).*(https?://\\S+).*')\ndef title_auto(bot, trigger):\n    \"\"\"\n    Automatically show titles for URLs. For shortened URLs/redirects, find\n    where the URL redirects to and show the title for that (or call a function\n    from another module to give more information).\n    \"\"\"\n    if re.match(bot.config.core.prefix + 'title', trigger):\n        return\n\n    # Avoid fetching known malicious links\n    if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:\n        if bot.memory['safety_cache'][trigger]['positives'] > 1:\n            return\n\n    urls = re.findall(url_finder, trigger)\n    if len(urls) == 0:\n        return\n\n    results = process_urls(bot, trigger, urls)\n    bot.memory['last_seen_url'][trigger.sender] = urls[-1]\n\n    for title, domain in results[:4]:\n        message = '[ %s ] - %s' % (title, domain)\n        # Guard against responding to other instances of this bot.\n        if message != trigger:\n            bot.say(message)\n\n\ndef process_urls(bot, trigger, urls):\n    \"\"\"\n    For each URL in the list, ensure that it isn't handled by another module.\n    If not, find where it redirects to, if anywhere. If that redirected URL\n    should be handled by another module, dispatch the callback for it.\n    Return a list of (title, hostname) tuples for each URL which is not handled by\n    another module.\n    \"\"\"\n\n    results = []\n    for url in urls:\n        if not url.startswith(bot.config.url.exclusion_char):\n            # Magic stuff to account for international domain names\n            try:\n                url = web.iri_to_uri(url)\n            except:\n                pass\n            # First, check that the URL we got doesn't match\n            matched = check_callbacks(bot, trigger, url, False)\n            if matched:\n                continue\n            # Finally, actually show the URL\n            title = find_title(url)\n            if title:\n                results.append((title, get_hostname(url)))\n    return results\n\n\ndef check_callbacks(bot, trigger, url, run=True):\n    \"\"\"\n    Check the given URL against the callbacks list. If it matches, and ``run``\n    is given as ``True``, run the callback function, otherwise pass. Returns\n    ``True`` if the url matched anything in the callbacks list.\n    \"\"\"\n    # Check if it matches the exclusion list first\n    matched = any(regex.search(url) for regex in bot.memory['url_exclude'])\n    # Then, check if there's anything in the callback list\n    for regex, function in tools.iteritems(bot.memory['url_callbacks']):\n        match = regex.search(url)\n        if match:\n            if run:\n                function(bot, trigger, match)\n            matched = True\n    return matched\n\n\ndef find_title(url):\n    \"\"\"Return the title for the given URL.\"\"\"\n    response = requests.get(url, stream=True)\n    try:\n        content = ''\n        for byte in response.iter_content(chunk_size=512, decode_unicode=True):\n            if not isinstance(byte, bytes):\n                content += byte\n            else:\n                break\n            if '</title>' in content or len(content) > max_bytes:\n                break\n    except UnicodeDecodeError:\n        return  # Fail silently when data can't be decoded\n    finally:\n        # need to close the connexion because we have not read all the data\n        response.close()\n\n    # Some cleanup that I don't really grok, but was in the original, so\n    # we'll keep it (with the compiled regexes made global) for now.\n    content = title_tag_data.sub(r'<\\1title>', content)\n    content = quoted_title.sub('', content)\n\n    start = content.find('<title>')\n    end = content.find('</title>')\n    if start == -1 or end == -1:\n        return\n    title = web.decode(content[start + 7:end])\n    title = title.strip()[:200]\n\n    title = ' '.join(title.split())  # cleanly remove multiple spaces\n\n    # More cryptic regex substitutions. This one looks to be myano's invention.\n    title = re_dcc.sub('', title)\n\n    return title or None\n\n\ndef get_hostname(url):\n    idx = 7\n    if url.startswith('https://'):\n        idx = 8\n    elif url.startswith('ftp://'):\n        idx = 6\n    hostname = url[idx:]\n    slash = hostname.find('/')\n    if slash != -1:\n        hostname = hostname[:slash]\n    return hostname\n\nif __name__ == \"__main__\":\n    from sopel.test_tools import run_example_tests\n    run_example_tests(__file__)\n", "path": "sopel/modules/url.py"}]}