We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Error in url setup procedure - Bot no longer posts titles of links
Hello. I've actually had this error for awhile and tried fixing it awhile back to no avail so I'm hoping someone here could help. The bot no longer seems to show titles of links posted in the channels and when starting the bot I get:

```
Error in url setup procedure: nothing to repeat (../../../../../lib/python2.7/sre_parse.py:517)
```

Which looking at that file points to this:

```
            if not item or (_len(item) == 1 and item[0][0] == AT):
                raise error, "nothing to repeat"
```

So I'm not sure what to do here.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `sopel/modules/url.py`
Content:
```
1 # coding=utf-8
2 """URL title module"""
3 # Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham
4 # Copyright 2012-2013 Elsie Powell
5 # Copyright 2013      Lior Ramati (firerogue517@gmail.com)
6 # Copyright © 2014 Elad Alfassa <elad@fedoraproject.org>
7 # Licensed under the Eiffel Forum License 2.
8 from __future__ import unicode_literals, absolute_import, print_function, division
9 
10 import re
11 from sopel import web, tools
12 from sopel.module import commands, rule, example
13 from sopel.config.types import ValidatedAttribute, StaticSection
14 
15 
16 url_finder = None
17 # These are used to clean up the title tag before actually parsing it. Not the
18 # world's best way to do this, but it'll do for now.
19 title_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)
20 quoted_title = re.compile('[\'"]<title>[\'"]', re.IGNORECASE)
21 # This is another regex that presumably does something important.
22 re_dcc = re.compile(r'(?i)dcc\ssend')
23 # This sets the maximum number of bytes that should be read in order to find
24 # the title. We don't want it too high, or a link to a big file/stream will
25 # just keep downloading until there's no more memory. 640k ought to be enough
26 # for anybody.
27 max_bytes = 655360
28 
29 
30 class UrlSection(StaticSection):
31     # TODO some validation rules maybe?
32     exclude = ValidatedAttribute('exclude')
33     exclusion_char = ValidatedAttribute('exclusion_char', default='!')
34 
35 
36 def configure(config):
37     config.define_section('url', UrlSection)
38     config.url.configure_setting(
39         'exclude',
40         'Enter regular expressions for each URL you would like to exclude.'
41     )
42     config.url.configure_setting(
43         'exclusion_char',
44         'Enter a character which can be prefixed to suppress URL titling'
45     )
46 
47 
48 def setup(bot=None):
49     global url_finder
50 
51     # TODO figure out why this is needed, and get rid of it, because really?
52     if not bot:
53         return
54     bot.config.define_section('url', UrlSection)
55 
56     if bot.config.url.exclude:
57         regexes = [re.compile(s) for s in bot.config.url.exclude]
58     else:
59         regexes = []
60 
61     # We're keeping these in their own list, rather than putting then in the
62     # callbacks list because 1, it's easier to deal with modules that are still
63     # using this list, and not the newer callbacks list and 2, having a lambda
64     # just to pass is kinda ugly.
65     if not bot.memory.contains('url_exclude'):
66         bot.memory['url_exclude'] = regexes
67     else:
68         exclude = bot.memory['url_exclude']
69         if regexes:
70             exclude.extend(regexes)
71         bot.memory['url_exclude'] = exclude
72 
73     # Ensure that url_callbacks and last_seen_url are in memory
74     if not bot.memory.contains('url_callbacks'):
75         bot.memory['url_callbacks'] = tools.SopelMemory()
76     if not bot.memory.contains('last_seen_url'):
77         bot.memory['last_seen_url'] = tools.SopelMemory()
78 
79     url_finder = re.compile(r'(?u)(%s?(?:http|https|ftp)(?:://\S+))' %
80                             (bot.config.url.exclusion_char))
81 
82 
83 @commands('title')
84 @example('.title http://google.com', '[ Google ] - google.com')
85 def title_command(bot, trigger):
86     """
87     Show the title or URL information for the given URL, or the last URL seen
88     in this channel.
89     """
90     if not trigger.group(2):
91         if trigger.sender not in bot.memory['last_seen_url']:
92             return
93         matched = check_callbacks(bot, trigger,
94                                   bot.memory['last_seen_url'][trigger.sender],
95                                   True)
96         if matched:
97             return
98         else:
99             urls = [bot.memory['last_seen_url'][trigger.sender]]
100     else:
101         urls = re.findall(url_finder, trigger)
102 
103     results = process_urls(bot, trigger, urls)
104     for title, domain in results[:4]:
105         bot.reply('[ %s ] - %s' % (title, domain))
106 
107 
108 @rule('(?u).*(https?://\S+).*')
109 def title_auto(bot, trigger):
110     """
111     Automatically show titles for URLs. For shortened URLs/redirects, find
112     where the URL redirects to and show the title for that (or call a function
113     from another module to give more information).
114     """
115     if re.match(bot.config.core.prefix + 'title', trigger):
116         return
117 
118     # Avoid fetching known malicious links
119     if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:
120         if bot.memory['safety_cache'][trigger]['positives'] > 1:
121             return
122 
123     urls = re.findall(url_finder, trigger)
124     results = process_urls(bot, trigger, urls)
125     bot.memory['last_seen_url'][trigger.sender] = urls[-1]
126 
127     for title, domain in results[:4]:
128         message = '[ %s ] - %s' % (title, domain)
129         # Guard against responding to other instances of this bot.
130         if message != trigger:
131             bot.say(message)
132 
133 
134 def process_urls(bot, trigger, urls):
135     """
136     For each URL in the list, ensure that it isn't handled by another module.
137     If not, find where it redirects to, if anywhere. If that redirected URL
138     should be handled by another module, dispatch the callback for it.
139     Return a list of (title, hostname) tuples for each URL which is not handled by
140     another module.
141     """
142 
143     results = []
144     for url in urls:
145         if not url.startswith(bot.config.url.exclusion_char):
146             # Magic stuff to account for international domain names
147             try:
148                 url = web.iri_to_uri(url)
149             except:
150                 pass
151             # First, check that the URL we got doesn't match
152             matched = check_callbacks(bot, trigger, url, False)
153             if matched:
154                 continue
155             # Then see if it redirects anywhere
156             new_url = follow_redirects(url)
157             if not new_url:
158                 continue
159             # Then see if the final URL matches anything
160             matched = check_callbacks(bot, trigger, new_url, new_url != url)
161             if matched:
162                 continue
163             # Finally, actually show the URL
164             title = find_title(url)
165             if title:
166                 results.append((title, get_hostname(url)))
167     return results
168 
169 
170 def follow_redirects(url):
171     """
172     Follow HTTP 3xx redirects, and return the actual URL. Return None if
173     there's a problem.
174     """
175     try:
176         connection = web.get_urllib_object(url, 60)
177         url = connection.geturl() or url
178         connection.close()
179     except:
180         return None
181     return url
182 
183 
184 def check_callbacks(bot, trigger, url, run=True):
185     """
186     Check the given URL against the callbacks list. If it matches, and ``run``
187     is given as ``True``, run the callback function, otherwise pass. Returns
188     ``True`` if the url matched anything in the callbacks list.
189     """
190     # Check if it matches the exclusion list first
191     matched = any(regex.search(url) for regex in bot.memory['url_exclude'])
192     # Then, check if there's anything in the callback list
193     for regex, function in tools.iteritems(bot.memory['url_callbacks']):
194         match = regex.search(url)
195         if match:
196             if run:
197                 function(bot, trigger, match)
198             matched = True
199     return matched
200 
201 
202 def find_title(url):
203     """Return the title for the given URL."""
204     try:
205         content, headers = web.get(url, return_headers=True, limit_bytes=max_bytes)
206     except UnicodeDecodeError:
207         return  # Fail silently when data can't be decoded
208 
209     # Some cleanup that I don't really grok, but was in the original, so
210     # we'll keep it (with the compiled regexes made global) for now.
211     content = title_tag_data.sub(r'<\1title>', content)
212     content = quoted_title.sub('', content)
213 
214     start = content.find('<title>')
215     end = content.find('</title>')
216     if start == -1 or end == -1:
217         return
218     title = web.decode(content[start + 7:end])
219     title = title.strip()[:200]
220 
221     title = ' '.join(title.split())  # cleanly remove multiple spaces
222 
223     # More cryptic regex substitutions. This one looks to be myano's invention.
224     title = re_dcc.sub('', title)
225 
226     return title or None
227 
228 
229 def get_hostname(url):
230     idx = 7
231     if url.startswith('https://'):
232         idx = 8
233     elif url.startswith('ftp://'):
234         idx = 6
235     hostname = url[idx:]
236     slash = hostname.find('/')
237     if slash != -1:
238         hostname = hostname[:slash]
239     return hostname
240 
241 if __name__ == "__main__":
242     from sopel.test_tools import run_example_tests
243     run_example_tests(__file__)
244
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/sopel/modules/url.py b/sopel/modules/url.py
--- a/sopel/modules/url.py
+++ b/sopel/modules/url.py
@@ -10,7 +10,7 @@
 import re
 from sopel import web, tools
 from sopel.module import commands, rule, example
-from sopel.config.types import ValidatedAttribute, StaticSection
+from sopel.config.types import ValidatedAttribute, ListAttribute, StaticSection
 
 
 url_finder = None
@@ -29,7 +29,7 @@
 
 class UrlSection(StaticSection):
     # TODO some validation rules maybe?
-    exclude = ValidatedAttribute('exclude')
+    exclude = ListAttribute('exclude')
     exclusion_char = ValidatedAttribute('exclusion_char', default='!')

{"golden_diff": "diff --git a/sopel/modules/url.py b/sopel/modules/url.py\n--- a/sopel/modules/url.py\n+++ b/sopel/modules/url.py\n@@ -10,7 +10,7 @@\n import re\n from sopel import web, tools\n from sopel.module import commands, rule, example\n-from sopel.config.types import ValidatedAttribute, StaticSection\n+from sopel.config.types import ValidatedAttribute, ListAttribute, StaticSection\n \n \n url_finder = None\n@@ -29,7 +29,7 @@\n \n class UrlSection(StaticSection):\n     # TODO some validation rules maybe?\n-    exclude = ValidatedAttribute('exclude')\n+    exclude = ListAttribute('exclude')\n     exclusion_char = ValidatedAttribute('exclusion_char', default='!')\n", "issue": "Error in url setup procedure - Bot no longer posts titles of links\nHello. I've actually had this error for awhile and tried fixing it awhile back to no avail so I'm hoping someone here could help. The bot no longer seems to show titles of links posted in the channels and when starting the bot I get:\n\n```\nError in url setup procedure: nothing to repeat (../../../../../lib/python2.7/sre_parse.py:517)\n```\n\nWhich looking at that file points to this:\n\n```\n            if not item or (_len(item) == 1 and item[0][0] == AT):\n                raise error, \"nothing to repeat\"\n```\n\nSo I'm not sure what to do here.\n\n", "before_files": [{"content": "# coding=utf-8\n\"\"\"URL title module\"\"\"\n# Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham\n# Copyright 2012-2013 Elsie Powell\n# Copyright 2013      Lior Ramati (firerogue517@gmail.com)\n# Copyright \u00a9 2014 Elad Alfassa <elad@fedoraproject.org>\n# Licensed under the Eiffel Forum License 2.\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport re\nfrom sopel import web, tools\nfrom sopel.module import commands, rule, example\nfrom sopel.config.types import ValidatedAttribute, StaticSection\n\n\nurl_finder = None\n# These are used to clean up the title tag before actually parsing it. Not the\n# world's best way to do this, but it'll do for now.\ntitle_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)\nquoted_title = re.compile('[\\'\"]<title>[\\'\"]', re.IGNORECASE)\n# This is another regex that presumably does something important.\nre_dcc = re.compile(r'(?i)dcc\\ssend')\n# This sets the maximum number of bytes that should be read in order to find\n# the title. We don't want it too high, or a link to a big file/stream will\n# just keep downloading until there's no more memory. 640k ought to be enough\n# for anybody.\nmax_bytes = 655360\n\n\nclass UrlSection(StaticSection):\n    # TODO some validation rules maybe?\n    exclude = ValidatedAttribute('exclude')\n    exclusion_char = ValidatedAttribute('exclusion_char', default='!')\n\n\ndef configure(config):\n    config.define_section('url', UrlSection)\n    config.url.configure_setting(\n        'exclude',\n        'Enter regular expressions for each URL you would like to exclude.'\n    )\n    config.url.configure_setting(\n        'exclusion_char',\n        'Enter a character which can be prefixed to suppress URL titling'\n    )\n\n\ndef setup(bot=None):\n    global url_finder\n\n    # TODO figure out why this is needed, and get rid of it, because really?\n    if not bot:\n        return\n    bot.config.define_section('url', UrlSection)\n\n    if bot.config.url.exclude:\n        regexes = [re.compile(s) for s in bot.config.url.exclude]\n    else:\n        regexes = []\n\n    # We're keeping these in their own list, rather than putting then in the\n    # callbacks list because 1, it's easier to deal with modules that are still\n    # using this list, and not the newer callbacks list and 2, having a lambda\n    # just to pass is kinda ugly.\n    if not bot.memory.contains('url_exclude'):\n        bot.memory['url_exclude'] = regexes\n    else:\n        exclude = bot.memory['url_exclude']\n        if regexes:\n            exclude.extend(regexes)\n        bot.memory['url_exclude'] = exclude\n\n    # Ensure that url_callbacks and last_seen_url are in memory\n    if not bot.memory.contains('url_callbacks'):\n        bot.memory['url_callbacks'] = tools.SopelMemory()\n    if not bot.memory.contains('last_seen_url'):\n        bot.memory['last_seen_url'] = tools.SopelMemory()\n\n    url_finder = re.compile(r'(?u)(%s?(?:http|https|ftp)(?:://\\S+))' %\n                            (bot.config.url.exclusion_char))\n\n\n@commands('title')\n@example('.title http://google.com', '[ Google ] - google.com')\ndef title_command(bot, trigger):\n    \"\"\"\n    Show the title or URL information for the given URL, or the last URL seen\n    in this channel.\n    \"\"\"\n    if not trigger.group(2):\n        if trigger.sender not in bot.memory['last_seen_url']:\n            return\n        matched = check_callbacks(bot, trigger,\n                                  bot.memory['last_seen_url'][trigger.sender],\n                                  True)\n        if matched:\n            return\n        else:\n            urls = [bot.memory['last_seen_url'][trigger.sender]]\n    else:\n        urls = re.findall(url_finder, trigger)\n\n    results = process_urls(bot, trigger, urls)\n    for title, domain in results[:4]:\n        bot.reply('[ %s ] - %s' % (title, domain))\n\n\n@rule('(?u).*(https?://\\S+).*')\ndef title_auto(bot, trigger):\n    \"\"\"\n    Automatically show titles for URLs. For shortened URLs/redirects, find\n    where the URL redirects to and show the title for that (or call a function\n    from another module to give more information).\n    \"\"\"\n    if re.match(bot.config.core.prefix + 'title', trigger):\n        return\n\n    # Avoid fetching known malicious links\n    if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:\n        if bot.memory['safety_cache'][trigger]['positives'] > 1:\n            return\n\n    urls = re.findall(url_finder, trigger)\n    results = process_urls(bot, trigger, urls)\n    bot.memory['last_seen_url'][trigger.sender] = urls[-1]\n\n    for title, domain in results[:4]:\n        message = '[ %s ] - %s' % (title, domain)\n        # Guard against responding to other instances of this bot.\n        if message != trigger:\n            bot.say(message)\n\n\ndef process_urls(bot, trigger, urls):\n    \"\"\"\n    For each URL in the list, ensure that it isn't handled by another module.\n    If not, find where it redirects to, if anywhere. If that redirected URL\n    should be handled by another module, dispatch the callback for it.\n    Return a list of (title, hostname) tuples for each URL which is not handled by\n    another module.\n    \"\"\"\n\n    results = []\n    for url in urls:\n        if not url.startswith(bot.config.url.exclusion_char):\n            # Magic stuff to account for international domain names\n            try:\n                url = web.iri_to_uri(url)\n            except:\n                pass\n            # First, check that the URL we got doesn't match\n            matched = check_callbacks(bot, trigger, url, False)\n            if matched:\n                continue\n            # Then see if it redirects anywhere\n            new_url = follow_redirects(url)\n            if not new_url:\n                continue\n            # Then see if the final URL matches anything\n            matched = check_callbacks(bot, trigger, new_url, new_url != url)\n            if matched:\n                continue\n            # Finally, actually show the URL\n            title = find_title(url)\n            if title:\n                results.append((title, get_hostname(url)))\n    return results\n\n\ndef follow_redirects(url):\n    \"\"\"\n    Follow HTTP 3xx redirects, and return the actual URL. Return None if\n    there's a problem.\n    \"\"\"\n    try:\n        connection = web.get_urllib_object(url, 60)\n        url = connection.geturl() or url\n        connection.close()\n    except:\n        return None\n    return url\n\n\ndef check_callbacks(bot, trigger, url, run=True):\n    \"\"\"\n    Check the given URL against the callbacks list. If it matches, and ``run``\n    is given as ``True``, run the callback function, otherwise pass. Returns\n    ``True`` if the url matched anything in the callbacks list.\n    \"\"\"\n    # Check if it matches the exclusion list first\n    matched = any(regex.search(url) for regex in bot.memory['url_exclude'])\n    # Then, check if there's anything in the callback list\n    for regex, function in tools.iteritems(bot.memory['url_callbacks']):\n        match = regex.search(url)\n        if match:\n            if run:\n                function(bot, trigger, match)\n            matched = True\n    return matched\n\n\ndef find_title(url):\n    \"\"\"Return the title for the given URL.\"\"\"\n    try:\n        content, headers = web.get(url, return_headers=True, limit_bytes=max_bytes)\n    except UnicodeDecodeError:\n        return  # Fail silently when data can't be decoded\n\n    # Some cleanup that I don't really grok, but was in the original, so\n    # we'll keep it (with the compiled regexes made global) for now.\n    content = title_tag_data.sub(r'<\\1title>', content)\n    content = quoted_title.sub('', content)\n\n    start = content.find('<title>')\n    end = content.find('</title>')\n    if start == -1 or end == -1:\n        return\n    title = web.decode(content[start + 7:end])\n    title = title.strip()[:200]\n\n    title = ' '.join(title.split())  # cleanly remove multiple spaces\n\n    # More cryptic regex substitutions. This one looks to be myano's invention.\n    title = re_dcc.sub('', title)\n\n    return title or None\n\n\ndef get_hostname(url):\n    idx = 7\n    if url.startswith('https://'):\n        idx = 8\n    elif url.startswith('ftp://'):\n        idx = 6\n    hostname = url[idx:]\n    slash = hostname.find('/')\n    if slash != -1:\n        hostname = hostname[:slash]\n    return hostname\n\nif __name__ == \"__main__\":\n    from sopel.test_tools import run_example_tests\n    run_example_tests(__file__)\n", "path": "sopel/modules/url.py"}], "after_files": [{"content": "# coding=utf-8\n\"\"\"URL title module\"\"\"\n# Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham\n# Copyright 2012-2013 Elsie Powell\n# Copyright 2013      Lior Ramati (firerogue517@gmail.com)\n# Copyright \u00a9 2014 Elad Alfassa <elad@fedoraproject.org>\n# Licensed under the Eiffel Forum License 2.\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport re\nfrom sopel import web, tools\nfrom sopel.module import commands, rule, example\nfrom sopel.config.types import ValidatedAttribute, ListAttribute, StaticSection\n\n\nurl_finder = None\n# These are used to clean up the title tag before actually parsing it. Not the\n# world's best way to do this, but it'll do for now.\ntitle_tag_data = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)\nquoted_title = re.compile('[\\'\"]<title>[\\'\"]', re.IGNORECASE)\n# This is another regex that presumably does something important.\nre_dcc = re.compile(r'(?i)dcc\\ssend')\n# This sets the maximum number of bytes that should be read in order to find\n# the title. We don't want it too high, or a link to a big file/stream will\n# just keep downloading until there's no more memory. 640k ought to be enough\n# for anybody.\nmax_bytes = 655360\n\n\nclass UrlSection(StaticSection):\n    # TODO some validation rules maybe?\n    exclude = ListAttribute('exclude')\n    exclusion_char = ValidatedAttribute('exclusion_char', default='!')\n\n\ndef configure(config):\n    config.define_section('url', UrlSection)\n    config.url.configure_setting(\n        'exclude',\n        'Enter regular expressions for each URL you would like to exclude.'\n    )\n    config.url.configure_setting(\n        'exclusion_char',\n        'Enter a character which can be prefixed to suppress URL titling'\n    )\n\n\ndef setup(bot=None):\n    global url_finder\n\n    # TODO figure out why this is needed, and get rid of it, because really?\n    if not bot:\n        return\n    bot.config.define_section('url', UrlSection)\n\n    if bot.config.url.exclude:\n        regexes = [re.compile(s) for s in bot.config.url.exclude]\n    else:\n        regexes = []\n\n    # We're keeping these in their own list, rather than putting then in the\n    # callbacks list because 1, it's easier to deal with modules that are still\n    # using this list, and not the newer callbacks list and 2, having a lambda\n    # just to pass is kinda ugly.\n    if not bot.memory.contains('url_exclude'):\n        bot.memory['url_exclude'] = regexes\n    else:\n        exclude = bot.memory['url_exclude']\n        if regexes:\n            exclude.extend(regexes)\n        bot.memory['url_exclude'] = exclude\n\n    # Ensure that url_callbacks and last_seen_url are in memory\n    if not bot.memory.contains('url_callbacks'):\n        bot.memory['url_callbacks'] = tools.SopelMemory()\n    if not bot.memory.contains('last_seen_url'):\n        bot.memory['last_seen_url'] = tools.SopelMemory()\n\n    url_finder = re.compile(r'(?u)(%s?(?:http|https|ftp)(?:://\\S+))' %\n                            (bot.config.url.exclusion_char))\n\n\n@commands('title')\n@example('.title http://google.com', '[ Google ] - google.com')\ndef title_command(bot, trigger):\n    \"\"\"\n    Show the title or URL information for the given URL, or the last URL seen\n    in this channel.\n    \"\"\"\n    if not trigger.group(2):\n        if trigger.sender not in bot.memory['last_seen_url']:\n            return\n        matched = check_callbacks(bot, trigger,\n                                  bot.memory['last_seen_url'][trigger.sender],\n                                  True)\n        if matched:\n            return\n        else:\n            urls = [bot.memory['last_seen_url'][trigger.sender]]\n    else:\n        urls = re.findall(url_finder, trigger)\n\n    results = process_urls(bot, trigger, urls)\n    for title, domain in results[:4]:\n        bot.reply('[ %s ] - %s' % (title, domain))\n\n\n@rule('(?u).*(https?://\\S+).*')\ndef title_auto(bot, trigger):\n    \"\"\"\n    Automatically show titles for URLs. For shortened URLs/redirects, find\n    where the URL redirects to and show the title for that (or call a function\n    from another module to give more information).\n    \"\"\"\n    if re.match(bot.config.core.prefix + 'title', trigger):\n        return\n\n    # Avoid fetching known malicious links\n    if 'safety_cache' in bot.memory and trigger in bot.memory['safety_cache']:\n        if bot.memory['safety_cache'][trigger]['positives'] > 1:\n            return\n\n    urls = re.findall(url_finder, trigger)\n    results = process_urls(bot, trigger, urls)\n    bot.memory['last_seen_url'][trigger.sender] = urls[-1]\n\n    for title, domain in results[:4]:\n        message = '[ %s ] - %s' % (title, domain)\n        # Guard against responding to other instances of this bot.\n        if message != trigger:\n            bot.say(message)\n\n\ndef process_urls(bot, trigger, urls):\n    \"\"\"\n    For each URL in the list, ensure that it isn't handled by another module.\n    If not, find where it redirects to, if anywhere. If that redirected URL\n    should be handled by another module, dispatch the callback for it.\n    Return a list of (title, hostname) tuples for each URL which is not handled by\n    another module.\n    \"\"\"\n\n    results = []\n    for url in urls:\n        if not url.startswith(bot.config.url.exclusion_char):\n            # Magic stuff to account for international domain names\n            try:\n                url = web.iri_to_uri(url)\n            except:\n                pass\n            # First, check that the URL we got doesn't match\n            matched = check_callbacks(bot, trigger, url, False)\n            if matched:\n                continue\n            # Then see if it redirects anywhere\n            new_url = follow_redirects(url)\n            if not new_url:\n                continue\n            # Then see if the final URL matches anything\n            matched = check_callbacks(bot, trigger, new_url, new_url != url)\n            if matched:\n                continue\n            # Finally, actually show the URL\n            title = find_title(url)\n            if title:\n                results.append((title, get_hostname(url)))\n    return results\n\n\ndef follow_redirects(url):\n    \"\"\"\n    Follow HTTP 3xx redirects, and return the actual URL. Return None if\n    there's a problem.\n    \"\"\"\n    try:\n        connection = web.get_urllib_object(url, 60)\n        url = connection.geturl() or url\n        connection.close()\n    except:\n        return None\n    return url\n\n\ndef check_callbacks(bot, trigger, url, run=True):\n    \"\"\"\n    Check the given URL against the callbacks list. If it matches, and ``run``\n    is given as ``True``, run the callback function, otherwise pass. Returns\n    ``True`` if the url matched anything in the callbacks list.\n    \"\"\"\n    # Check if it matches the exclusion list first\n    matched = any(regex.search(url) for regex in bot.memory['url_exclude'])\n    # Then, check if there's anything in the callback list\n    for regex, function in tools.iteritems(bot.memory['url_callbacks']):\n        match = regex.search(url)\n        if match:\n            if run:\n                function(bot, trigger, match)\n            matched = True\n    return matched\n\n\ndef find_title(url):\n    \"\"\"Return the title for the given URL.\"\"\"\n    try:\n        content, headers = web.get(url, return_headers=True, limit_bytes=max_bytes)\n    except UnicodeDecodeError:\n        return  # Fail silently when data can't be decoded\n\n    # Some cleanup that I don't really grok, but was in the original, so\n    # we'll keep it (with the compiled regexes made global) for now.\n    content = title_tag_data.sub(r'<\\1title>', content)\n    content = quoted_title.sub('', content)\n\n    start = content.find('<title>')\n    end = content.find('</title>')\n    if start == -1 or end == -1:\n        return\n    title = web.decode(content[start + 7:end])\n    title = title.strip()[:200]\n\n    title = ' '.join(title.split())  # cleanly remove multiple spaces\n\n    # More cryptic regex substitutions. This one looks to be myano's invention.\n    title = re_dcc.sub('', title)\n\n    return title or None\n\n\ndef get_hostname(url):\n    idx = 7\n    if url.startswith('https://'):\n        idx = 8\n    elif url.startswith('ftp://'):\n        idx = 6\n    hostname = url[idx:]\n    slash = hostname.find('/')\n    if slash != -1:\n        hostname = hostname[:slash]\n    return hostname\n\nif __name__ == \"__main__\":\n    from sopel.test_tools import run_example_tests\n    run_example_tests(__file__)\n", "path": "sopel/modules/url.py"}]}