We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Safety throws error with positives
<!-- Before reporting a bug, please search both open *and closed* issues to
see if it has already been reported. If you can, try to reproduce the problem
on an unmodified copy of the `master` branch first, as sometimes bugs are found
and fixed without a report. If the problem is unreported and persists in
`master`, please help us fix it quickly by filling out as much of this
information as you can. Thanks! -->

### Description
When Safety is enabled with a valid personal free API key whenever a new URL is seen by the bot in a safety enabled channel, it throws an error.

### Reproduction steps
1. Set `safety` to `on` or `strict`
2. Send new message containing a URL

### Expected behavior
What did you expect to happen instead?

### Logs
From the bot
```
<Ad> https://www.cnn.com/2020/04/21/uk/uk-death-toll-coronavirus-statistics-gbr-intl/index.html
<^quark^> [ UK coronavirus: Death toll in England and Wales 41% higher than government figures - CNN ] - www.cnn.com
<^quark^> Unexpected error ('positives') from Ad at 2020-04-21 21:47:29.752477. Message was: https://www.cnn.com/2020/04/21/uk/uk-death-toll-coronavirus-statistics-gbr-intl/index.html
```

From logs
```
[2020-04-21 21:47:29,752] sopel.bot            ERROR    - Unexpected error ('positives') from Ad at 2020-04-21 21:47:29.752477. Message was: https://www.cnn.com/2020/04/21/uk/uk-death-toll-coronavirus-statistics-gbr-intl/index.html
Traceback (most recent call last):
  File "/home/sopel/.local/lib/python3.8/site-packages/sopel-7.0.0-py3.8.egg/sopel/bot.py", line 590, in call
    exit_code = func(sopel, trigger)
  File "/home/sopel/.local/lib/python3.8/site-packages/sopel-7.0.0-py3.8.egg/sopel/modules/safety.py", line 166, in url_handler
    data = {'positives': result['positives'],
KeyError: 'positives'

```

### Environment
- Sopel `.version`: 7.0.1
- Sopel installed via: git 
- Python version: 3.8
- Operating system: alpine
- IRCd `/version`: UnrealIRCd-5.0.3.1
- Relevant plugins: safety

### Notes



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `sopel/modules/safety.py`
Content:
```
1 # coding=utf-8
2 """
3 safety.py - Alerts about malicious URLs
4 Copyright © 2014, Elad Alfassa, <elad@fedoraproject.org>
5 Licensed under the Eiffel Forum License 2.
6 
7 This module uses virustotal.com
8 """
9 from __future__ import unicode_literals, absolute_import, print_function, division
10 
11 import logging
12 import os.path
13 import re
14 import sys
15 import threading
16 import time
17 
18 import requests
19 
20 from sopel.config.types import StaticSection, ValidatedAttribute, ListAttribute
21 from sopel.formatting import color, bold
22 from sopel.module import OP
23 import sopel.tools
24 
25 try:
26     # This is done separately from the below version if/else because JSONDecodeError
27     # didn't appear until Python 3.5, but Sopel claims support for 3.3+
28     # Redo this whole block of nonsense when dropping py2/old py3 support
29     from json import JSONDecodeError as InvalidJSONResponse
30 except ImportError:
31     InvalidJSONResponse = ValueError
32 
33 if sys.version_info.major > 2:
34     unicode = str
35     from urllib.request import urlretrieve
36     from urllib.parse import urlparse
37 else:
38     from urllib import urlretrieve
39     from urlparse import urlparse
40 
41 
42 LOGGER = logging.getLogger(__name__)
43 
44 vt_base_api_url = 'https://www.virustotal.com/vtapi/v2/url/'
45 malware_domains = set()
46 known_good = []
47 cache_limit = 512
48 
49 
50 class SafetySection(StaticSection):
51     enabled_by_default = ValidatedAttribute('enabled_by_default', bool, default=True)
52     """Whether to enable URL safety in all channels where it isn't explicitly disabled."""
53     known_good = ListAttribute('known_good')
54     """List of "known good" domains to ignore."""
55     vt_api_key = ValidatedAttribute('vt_api_key')
56     """Optional VirusTotal API key (improves malicious URL detection)."""
57 
58 
59 def configure(config):
60     """
61     | name | example | purpose |
62     | ---- | ------- | ------- |
63     | enabled\\_by\\_default | True | Enable URL safety in all channels where it isn't explicitly disabled. |
64     | known\\_good | sopel.chat,dftba.net | List of "known good" domains to ignore. |
65     | vt\\_api\\_key | 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef | Optional VirusTotal API key to improve malicious URL detection |
66     """
67     config.define_section('safety', SafetySection)
68     config.safety.configure_setting(
69         'enabled_by_default',
70         "Enable URL safety in channels that don't specifically disable it?",
71     )
72     config.safety.configure_setting(
73         'known_good',
74         'Enter any domains to whitelist',
75     )
76     config.safety.configure_setting(
77         'vt_api_key',
78         "Optionally, enter a VirusTotal API key to improve malicious URL "
79         "protection.\nOtherwise, only the Malwarebytes DB will be used."
80     )
81 
82 
83 def setup(bot):
84     bot.config.define_section('safety', SafetySection)
85 
86     if 'safety_cache' not in bot.memory:
87         bot.memory['safety_cache'] = sopel.tools.SopelMemory()
88     if 'safety_cache_lock' not in bot.memory:
89         bot.memory['safety_cache_lock'] = threading.Lock()
90     for item in bot.config.safety.known_good:
91         known_good.append(re.compile(item, re.I))
92 
93     loc = os.path.join(bot.config.homedir, 'malwaredomains.txt')
94     if os.path.isfile(loc):
95         if os.path.getmtime(loc) < time.time() - 24 * 60 * 60 * 7:
96             # File exists but older than one week — update it
97             _download_malwaredomains_db(loc)
98     else:
99         _download_malwaredomains_db(loc)
100     with open(loc, 'r') as f:
101         for line in f:
102             clean_line = unicode(line).strip().lower()
103             if clean_line != '':
104                 malware_domains.add(clean_line)
105 
106 
107 def shutdown(bot):
108     bot.memory.pop('safety_cache', None)
109     bot.memory.pop('safety_cache_lock', None)
110 
111 
112 def _download_malwaredomains_db(path):
113     url = 'https://mirror1.malwaredomains.com/files/justdomains'
114     LOGGER.info('Downloading malwaredomains db from %s', url)
115     urlretrieve(url, path)
116 
117 
118 @sopel.module.rule(r'(?u).*(https?://\S+).*')
119 @sopel.module.priority('high')
120 def url_handler(bot, trigger):
121     """Checks for malicious URLs"""
122     check = True    # Enable URL checking
123     strict = False  # Strict mode: kick on malicious URL
124     positives = 0   # Number of engines saying it's malicious
125     total = 0       # Number of total engines
126     use_vt = True   # Use VirusTotal
127     check = bot.config.safety.enabled_by_default
128     if check is None:
129         # If not set, assume default
130         check = True
131     # DB overrides config:
132     setting = bot.db.get_channel_value(trigger.sender, 'safety')
133     if setting is not None:
134         if setting == 'off':
135             return  # Not checking
136         elif setting in ['on', 'strict', 'local', 'local strict']:
137             check = True
138         if setting == 'strict' or setting == 'local strict':
139             strict = True
140         if setting == 'local' or setting == 'local strict':
141             use_vt = False
142 
143     if not check:
144         return  # Not overridden by DB, configured default off
145 
146     try:
147         netloc = urlparse(trigger.group(1)).netloc
148     except ValueError:
149         return  # Invalid IPv6 URL
150 
151     if any(regex.search(netloc) for regex in known_good):
152         return  # Whitelisted
153 
154     apikey = bot.config.safety.vt_api_key
155     try:
156         if apikey is not None and use_vt:
157             payload = {'resource': unicode(trigger),
158                        'apikey': apikey,
159                        'scan': '1'}
160 
161             if trigger not in bot.memory['safety_cache']:
162                 r = requests.post(vt_base_api_url + 'report', data=payload)
163                 r.raise_for_status()
164                 result = r.json()
165                 fetched = time.time()
166                 data = {'positives': result['positives'],
167                         'total': result['total'],
168                         'fetched': fetched}
169                 bot.memory['safety_cache'][trigger] = data
170                 if len(bot.memory['safety_cache']) >= (2 * cache_limit):
171                     _clean_cache(bot)
172             else:
173                 print('using cache')
174                 result = bot.memory['safety_cache'][trigger]
175             positives = result['positives']
176             total = result['total']
177     except requests.exceptions.RequestException:
178         # Ignoring exceptions with VT so MalwareDomains will always work
179         LOGGER.debug('[VirusTotal] Error obtaining response.', exc_info=True)
180     except InvalidJSONResponse:
181         # Ignoring exceptions with VT so MalwareDomains will always work
182         LOGGER.debug('[VirusTotal] Malformed response (invalid JSON).', exc_info=True)
183 
184     if unicode(netloc).lower() in malware_domains:
185         # malwaredomains is more trustworthy than some VT engines
186         # therefore it gets a weight of 10 engines when calculating confidence
187         positives += 10
188         total += 10
189 
190     if positives > 1:
191         # Possibly malicious URL detected!
192         confidence = '{}%'.format(round((positives / total) * 100))
193         msg = 'link posted by %s is possibly malicious ' % bold(trigger.nick)
194         msg += '(confidence %s - %s/%s)' % (confidence, positives, total)
195         bot.say('[' + bold(color('WARNING', 'red')) + '] ' + msg)
196         if strict:
197             bot.kick(trigger.nick, trigger.sender, 'Posted a malicious link')
198 
199 
200 @sopel.module.commands('safety')
201 def toggle_safety(bot, trigger):
202     """Set safety setting for channel"""
203     if not trigger.admin and bot.channels[trigger.sender].privileges[trigger.nick] < OP:
204         bot.reply('Only channel operators can change safety settings')
205         return
206     allowed_states = ['strict', 'on', 'off', 'local', 'local strict']
207     if not trigger.group(2) or trigger.group(2).lower() not in allowed_states:
208         options = ' / '.join(allowed_states)
209         bot.reply('Available options: %s' % options)
210         return
211 
212     channel = trigger.sender.lower()
213     bot.db.set_channel_value(channel, 'safety', trigger.group(2).lower())
214     bot.reply('Safety is now set to "%s" on this channel' % trigger.group(2))
215 
216 
217 # Clean the cache every day
218 # Code above also calls this if there are too many cache entries
219 @sopel.module.interval(24 * 60 * 60)
220 def _clean_cache(bot):
221     """Cleans up old entries in URL safety cache."""
222     if bot.memory['safety_cache_lock'].acquire(False):
223         LOGGER.info('Starting safety cache cleanup...')
224         try:
225             # clean up by age first
226             cutoff = time.time() - (7 * 24 * 60 * 60)  # 7 days ago
227             old_keys = []
228             for key, data in sopel.tools.iteritems(bot.memory['safety_cache']):
229                 if data['fetched'] <= cutoff:
230                     old_keys.append(key)
231             for key in old_keys:
232                 bot.memory['safety_cache'].pop(key, None)
233 
234             # clean up more values if the cache is still too big
235             overage = len(bot.memory['safety_cache']) - cache_limit
236             if overage > 0:
237                 extra_keys = sorted(
238                     (data.fetched, key)
239                     for (key, data)
240                     in bot.memory['safety_cache'].items())[:overage]
241                 for (_, key) in extra_keys:
242                     bot.memory['safety_cache'].pop(key, None)
243         finally:
244             # No matter what errors happen (or not), release the lock
245             bot.memory['safety_cache_lock'].release()
246 
247         LOGGER.info('Safety cache cleanup finished.')
248     else:
249         LOGGER.info(
250             'Skipping safety cache cleanup: Cache is locked, '
251             'cleanup already running.')
252
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/sopel/modules/safety.py b/sopel/modules/safety.py
--- a/sopel/modules/safety.py
+++ b/sopel/modules/safety.py
@@ -163,17 +163,20 @@
                 r.raise_for_status()
                 result = r.json()
                 fetched = time.time()
-                data = {'positives': result['positives'],
-                        'total': result['total'],
-                        'fetched': fetched}
-                bot.memory['safety_cache'][trigger] = data
-                if len(bot.memory['safety_cache']) >= (2 * cache_limit):
-                    _clean_cache(bot)
+                if all(k in result for k in ['positives', 'total']):
+                    # cache result only if it contains a scan report
+                    # TODO: handle checking back for results from queued scans
+                    data = {'positives': result['positives'],
+                            'total': result['total'],
+                            'fetched': fetched}
+                    bot.memory['safety_cache'][trigger] = data
+                    if len(bot.memory['safety_cache']) >= (2 * cache_limit):
+                        _clean_cache(bot)
             else:
                 print('using cache')
                 result = bot.memory['safety_cache'][trigger]
-            positives = result['positives']
-            total = result['total']
+            positives = result.get('positives', 0)
+            total = result.get('total', 0)
     except requests.exceptions.RequestException:
         # Ignoring exceptions with VT so MalwareDomains will always work
         LOGGER.debug('[VirusTotal] Error obtaining response.', exc_info=True)

{"golden_diff": "diff --git a/sopel/modules/safety.py b/sopel/modules/safety.py\n--- a/sopel/modules/safety.py\n+++ b/sopel/modules/safety.py\n@@ -163,17 +163,20 @@\n                 r.raise_for_status()\n                 result = r.json()\n                 fetched = time.time()\n-                data = {'positives': result['positives'],\n-                        'total': result['total'],\n-                        'fetched': fetched}\n-                bot.memory['safety_cache'][trigger] = data\n-                if len(bot.memory['safety_cache']) >= (2 * cache_limit):\n-                    _clean_cache(bot)\n+                if all(k in result for k in ['positives', 'total']):\n+                    # cache result only if it contains a scan report\n+                    # TODO: handle checking back for results from queued scans\n+                    data = {'positives': result['positives'],\n+                            'total': result['total'],\n+                            'fetched': fetched}\n+                    bot.memory['safety_cache'][trigger] = data\n+                    if len(bot.memory['safety_cache']) >= (2 * cache_limit):\n+                        _clean_cache(bot)\n             else:\n                 print('using cache')\n                 result = bot.memory['safety_cache'][trigger]\n-            positives = result['positives']\n-            total = result['total']\n+            positives = result.get('positives', 0)\n+            total = result.get('total', 0)\n     except requests.exceptions.RequestException:\n         # Ignoring exceptions with VT so MalwareDomains will always work\n         LOGGER.debug('[VirusTotal] Error obtaining response.', exc_info=True)\n", "issue": "Safety throws error with positives\n<!-- Before reporting a bug, please search both open *and closed* issues to\r\nsee if it has already been reported. If you can, try to reproduce the problem\r\non an unmodified copy of the `master` branch first, as sometimes bugs are found\r\nand fixed without a report. If the problem is unreported and persists in\r\n`master`, please help us fix it quickly by filling out as much of this\r\ninformation as you can. Thanks! -->\r\n\r\n### Description\r\nWhen Safety is enabled with a valid personal free API key whenever a new URL is seen by the bot in a safety enabled channel, it throws an error.\r\n\r\n### Reproduction steps\r\n1. Set `safety` to `on` or `strict`\r\n2. Send new message containing a URL\r\n\r\n### Expected behavior\r\nWhat did you expect to happen instead?\r\n\r\n### Logs\r\nFrom the bot\r\n```\r\n<Ad> https://www.cnn.com/2020/04/21/uk/uk-death-toll-coronavirus-statistics-gbr-intl/index.html\r\n<^quark^> [ UK coronavirus: Death toll in England and Wales 41% higher than government figures - CNN ] - www.cnn.com\r\n<^quark^> Unexpected error ('positives') from Ad at 2020-04-21 21:47:29.752477. Message was: https://www.cnn.com/2020/04/21/uk/uk-death-toll-coronavirus-statistics-gbr-intl/index.html\r\n```\r\n\r\nFrom logs\r\n```\r\n[2020-04-21 21:47:29,752] sopel.bot            ERROR    - Unexpected error ('positives') from Ad at 2020-04-21 21:47:29.752477. Message was: https://www.cnn.com/2020/04/21/uk/uk-death-toll-coronavirus-statistics-gbr-intl/index.html\r\nTraceback (most recent call last):\r\n  File \"/home/sopel/.local/lib/python3.8/site-packages/sopel-7.0.0-py3.8.egg/sopel/bot.py\", line 590, in call\r\n    exit_code = func(sopel, trigger)\r\n  File \"/home/sopel/.local/lib/python3.8/site-packages/sopel-7.0.0-py3.8.egg/sopel/modules/safety.py\", line 166, in url_handler\r\n    data = {'positives': result['positives'],\r\nKeyError: 'positives'\r\n\r\n```\r\n\r\n### Environment\r\n- Sopel `.version`: 7.0.1\r\n- Sopel installed via: git \r\n- Python version: 3.8\r\n- Operating system: alpine\r\n- IRCd `/version`: UnrealIRCd-5.0.3.1\r\n- Relevant plugins: safety\r\n\r\n### Notes\r\n\r\n\n", "before_files": [{"content": "# coding=utf-8\n\"\"\"\nsafety.py - Alerts about malicious URLs\nCopyright \u00a9 2014, Elad Alfassa, <elad@fedoraproject.org>\nLicensed under the Eiffel Forum License 2.\n\nThis module uses virustotal.com\n\"\"\"\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport logging\nimport os.path\nimport re\nimport sys\nimport threading\nimport time\n\nimport requests\n\nfrom sopel.config.types import StaticSection, ValidatedAttribute, ListAttribute\nfrom sopel.formatting import color, bold\nfrom sopel.module import OP\nimport sopel.tools\n\ntry:\n    # This is done separately from the below version if/else because JSONDecodeError\n    # didn't appear until Python 3.5, but Sopel claims support for 3.3+\n    # Redo this whole block of nonsense when dropping py2/old py3 support\n    from json import JSONDecodeError as InvalidJSONResponse\nexcept ImportError:\n    InvalidJSONResponse = ValueError\n\nif sys.version_info.major > 2:\n    unicode = str\n    from urllib.request import urlretrieve\n    from urllib.parse import urlparse\nelse:\n    from urllib import urlretrieve\n    from urlparse import urlparse\n\n\nLOGGER = logging.getLogger(__name__)\n\nvt_base_api_url = 'https://www.virustotal.com/vtapi/v2/url/'\nmalware_domains = set()\nknown_good = []\ncache_limit = 512\n\n\nclass SafetySection(StaticSection):\n    enabled_by_default = ValidatedAttribute('enabled_by_default', bool, default=True)\n    \"\"\"Whether to enable URL safety in all channels where it isn't explicitly disabled.\"\"\"\n    known_good = ListAttribute('known_good')\n    \"\"\"List of \"known good\" domains to ignore.\"\"\"\n    vt_api_key = ValidatedAttribute('vt_api_key')\n    \"\"\"Optional VirusTotal API key (improves malicious URL detection).\"\"\"\n\n\ndef configure(config):\n    \"\"\"\n    | name | example | purpose |\n    | ---- | ------- | ------- |\n    | enabled\\\\_by\\\\_default | True | Enable URL safety in all channels where it isn't explicitly disabled. |\n    | known\\\\_good | sopel.chat,dftba.net | List of \"known good\" domains to ignore. |\n    | vt\\\\_api\\\\_key | 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef | Optional VirusTotal API key to improve malicious URL detection |\n    \"\"\"\n    config.define_section('safety', SafetySection)\n    config.safety.configure_setting(\n        'enabled_by_default',\n        \"Enable URL safety in channels that don't specifically disable it?\",\n    )\n    config.safety.configure_setting(\n        'known_good',\n        'Enter any domains to whitelist',\n    )\n    config.safety.configure_setting(\n        'vt_api_key',\n        \"Optionally, enter a VirusTotal API key to improve malicious URL \"\n        \"protection.\\nOtherwise, only the Malwarebytes DB will be used.\"\n    )\n\n\ndef setup(bot):\n    bot.config.define_section('safety', SafetySection)\n\n    if 'safety_cache' not in bot.memory:\n        bot.memory['safety_cache'] = sopel.tools.SopelMemory()\n    if 'safety_cache_lock' not in bot.memory:\n        bot.memory['safety_cache_lock'] = threading.Lock()\n    for item in bot.config.safety.known_good:\n        known_good.append(re.compile(item, re.I))\n\n    loc = os.path.join(bot.config.homedir, 'malwaredomains.txt')\n    if os.path.isfile(loc):\n        if os.path.getmtime(loc) < time.time() - 24 * 60 * 60 * 7:\n            # File exists but older than one week \u2014 update it\n            _download_malwaredomains_db(loc)\n    else:\n        _download_malwaredomains_db(loc)\n    with open(loc, 'r') as f:\n        for line in f:\n            clean_line = unicode(line).strip().lower()\n            if clean_line != '':\n                malware_domains.add(clean_line)\n\n\ndef shutdown(bot):\n    bot.memory.pop('safety_cache', None)\n    bot.memory.pop('safety_cache_lock', None)\n\n\ndef _download_malwaredomains_db(path):\n    url = 'https://mirror1.malwaredomains.com/files/justdomains'\n    LOGGER.info('Downloading malwaredomains db from %s', url)\n    urlretrieve(url, path)\n\n\n@sopel.module.rule(r'(?u).*(https?://\\S+).*')\n@sopel.module.priority('high')\ndef url_handler(bot, trigger):\n    \"\"\"Checks for malicious URLs\"\"\"\n    check = True    # Enable URL checking\n    strict = False  # Strict mode: kick on malicious URL\n    positives = 0   # Number of engines saying it's malicious\n    total = 0       # Number of total engines\n    use_vt = True   # Use VirusTotal\n    check = bot.config.safety.enabled_by_default\n    if check is None:\n        # If not set, assume default\n        check = True\n    # DB overrides config:\n    setting = bot.db.get_channel_value(trigger.sender, 'safety')\n    if setting is not None:\n        if setting == 'off':\n            return  # Not checking\n        elif setting in ['on', 'strict', 'local', 'local strict']:\n            check = True\n        if setting == 'strict' or setting == 'local strict':\n            strict = True\n        if setting == 'local' or setting == 'local strict':\n            use_vt = False\n\n    if not check:\n        return  # Not overridden by DB, configured default off\n\n    try:\n        netloc = urlparse(trigger.group(1)).netloc\n    except ValueError:\n        return  # Invalid IPv6 URL\n\n    if any(regex.search(netloc) for regex in known_good):\n        return  # Whitelisted\n\n    apikey = bot.config.safety.vt_api_key\n    try:\n        if apikey is not None and use_vt:\n            payload = {'resource': unicode(trigger),\n                       'apikey': apikey,\n                       'scan': '1'}\n\n            if trigger not in bot.memory['safety_cache']:\n                r = requests.post(vt_base_api_url + 'report', data=payload)\n                r.raise_for_status()\n                result = r.json()\n                fetched = time.time()\n                data = {'positives': result['positives'],\n                        'total': result['total'],\n                        'fetched': fetched}\n                bot.memory['safety_cache'][trigger] = data\n                if len(bot.memory['safety_cache']) >= (2 * cache_limit):\n                    _clean_cache(bot)\n            else:\n                print('using cache')\n                result = bot.memory['safety_cache'][trigger]\n            positives = result['positives']\n            total = result['total']\n    except requests.exceptions.RequestException:\n        # Ignoring exceptions with VT so MalwareDomains will always work\n        LOGGER.debug('[VirusTotal] Error obtaining response.', exc_info=True)\n    except InvalidJSONResponse:\n        # Ignoring exceptions with VT so MalwareDomains will always work\n        LOGGER.debug('[VirusTotal] Malformed response (invalid JSON).', exc_info=True)\n\n    if unicode(netloc).lower() in malware_domains:\n        # malwaredomains is more trustworthy than some VT engines\n        # therefore it gets a weight of 10 engines when calculating confidence\n        positives += 10\n        total += 10\n\n    if positives > 1:\n        # Possibly malicious URL detected!\n        confidence = '{}%'.format(round((positives / total) * 100))\n        msg = 'link posted by %s is possibly malicious ' % bold(trigger.nick)\n        msg += '(confidence %s - %s/%s)' % (confidence, positives, total)\n        bot.say('[' + bold(color('WARNING', 'red')) + '] ' + msg)\n        if strict:\n            bot.kick(trigger.nick, trigger.sender, 'Posted a malicious link')\n\n\n@sopel.module.commands('safety')\ndef toggle_safety(bot, trigger):\n    \"\"\"Set safety setting for channel\"\"\"\n    if not trigger.admin and bot.channels[trigger.sender].privileges[trigger.nick] < OP:\n        bot.reply('Only channel operators can change safety settings')\n        return\n    allowed_states = ['strict', 'on', 'off', 'local', 'local strict']\n    if not trigger.group(2) or trigger.group(2).lower() not in allowed_states:\n        options = ' / '.join(allowed_states)\n        bot.reply('Available options: %s' % options)\n        return\n\n    channel = trigger.sender.lower()\n    bot.db.set_channel_value(channel, 'safety', trigger.group(2).lower())\n    bot.reply('Safety is now set to \"%s\" on this channel' % trigger.group(2))\n\n\n# Clean the cache every day\n# Code above also calls this if there are too many cache entries\n@sopel.module.interval(24 * 60 * 60)\ndef _clean_cache(bot):\n    \"\"\"Cleans up old entries in URL safety cache.\"\"\"\n    if bot.memory['safety_cache_lock'].acquire(False):\n        LOGGER.info('Starting safety cache cleanup...')\n        try:\n            # clean up by age first\n            cutoff = time.time() - (7 * 24 * 60 * 60)  # 7 days ago\n            old_keys = []\n            for key, data in sopel.tools.iteritems(bot.memory['safety_cache']):\n                if data['fetched'] <= cutoff:\n                    old_keys.append(key)\n            for key in old_keys:\n                bot.memory['safety_cache'].pop(key, None)\n\n            # clean up more values if the cache is still too big\n            overage = len(bot.memory['safety_cache']) - cache_limit\n            if overage > 0:\n                extra_keys = sorted(\n                    (data.fetched, key)\n                    for (key, data)\n                    in bot.memory['safety_cache'].items())[:overage]\n                for (_, key) in extra_keys:\n                    bot.memory['safety_cache'].pop(key, None)\n        finally:\n            # No matter what errors happen (or not), release the lock\n            bot.memory['safety_cache_lock'].release()\n\n        LOGGER.info('Safety cache cleanup finished.')\n    else:\n        LOGGER.info(\n            'Skipping safety cache cleanup: Cache is locked, '\n            'cleanup already running.')\n", "path": "sopel/modules/safety.py"}], "after_files": [{"content": "# coding=utf-8\n\"\"\"\nsafety.py - Alerts about malicious URLs\nCopyright \u00a9 2014, Elad Alfassa, <elad@fedoraproject.org>\nLicensed under the Eiffel Forum License 2.\n\nThis module uses virustotal.com\n\"\"\"\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport logging\nimport os.path\nimport re\nimport sys\nimport threading\nimport time\n\nimport requests\n\nfrom sopel.config.types import StaticSection, ValidatedAttribute, ListAttribute\nfrom sopel.formatting import color, bold\nfrom sopel.module import OP\nimport sopel.tools\n\ntry:\n    # This is done separately from the below version if/else because JSONDecodeError\n    # didn't appear until Python 3.5, but Sopel claims support for 3.3+\n    # Redo this whole block of nonsense when dropping py2/old py3 support\n    from json import JSONDecodeError as InvalidJSONResponse\nexcept ImportError:\n    InvalidJSONResponse = ValueError\n\nif sys.version_info.major > 2:\n    unicode = str\n    from urllib.request import urlretrieve\n    from urllib.parse import urlparse\nelse:\n    from urllib import urlretrieve\n    from urlparse import urlparse\n\n\nLOGGER = logging.getLogger(__name__)\n\nvt_base_api_url = 'https://www.virustotal.com/vtapi/v2/url/'\nmalware_domains = set()\nknown_good = []\ncache_limit = 512\n\n\nclass SafetySection(StaticSection):\n    enabled_by_default = ValidatedAttribute('enabled_by_default', bool, default=True)\n    \"\"\"Whether to enable URL safety in all channels where it isn't explicitly disabled.\"\"\"\n    known_good = ListAttribute('known_good')\n    \"\"\"List of \"known good\" domains to ignore.\"\"\"\n    vt_api_key = ValidatedAttribute('vt_api_key')\n    \"\"\"Optional VirusTotal API key (improves malicious URL detection).\"\"\"\n\n\ndef configure(config):\n    \"\"\"\n    | name | example | purpose |\n    | ---- | ------- | ------- |\n    | enabled\\\\_by\\\\_default | True | Enable URL safety in all channels where it isn't explicitly disabled. |\n    | known\\\\_good | sopel.chat,dftba.net | List of \"known good\" domains to ignore. |\n    | vt\\\\_api\\\\_key | 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef | Optional VirusTotal API key to improve malicious URL detection |\n    \"\"\"\n    config.define_section('safety', SafetySection)\n    config.safety.configure_setting(\n        'enabled_by_default',\n        \"Enable URL safety in channels that don't specifically disable it?\",\n    )\n    config.safety.configure_setting(\n        'known_good',\n        'Enter any domains to whitelist',\n    )\n    config.safety.configure_setting(\n        'vt_api_key',\n        \"Optionally, enter a VirusTotal API key to improve malicious URL \"\n        \"protection.\\nOtherwise, only the Malwarebytes DB will be used.\"\n    )\n\n\ndef setup(bot):\n    bot.config.define_section('safety', SafetySection)\n\n    if 'safety_cache' not in bot.memory:\n        bot.memory['safety_cache'] = sopel.tools.SopelMemory()\n    if 'safety_cache_lock' not in bot.memory:\n        bot.memory['safety_cache_lock'] = threading.Lock()\n    for item in bot.config.safety.known_good:\n        known_good.append(re.compile(item, re.I))\n\n    loc = os.path.join(bot.config.homedir, 'malwaredomains.txt')\n    if os.path.isfile(loc):\n        if os.path.getmtime(loc) < time.time() - 24 * 60 * 60 * 7:\n            # File exists but older than one week \u2014 update it\n            _download_malwaredomains_db(loc)\n    else:\n        _download_malwaredomains_db(loc)\n    with open(loc, 'r') as f:\n        for line in f:\n            clean_line = unicode(line).strip().lower()\n            if clean_line != '':\n                malware_domains.add(clean_line)\n\n\ndef shutdown(bot):\n    bot.memory.pop('safety_cache', None)\n    bot.memory.pop('safety_cache_lock', None)\n\n\ndef _download_malwaredomains_db(path):\n    url = 'https://mirror1.malwaredomains.com/files/justdomains'\n    LOGGER.info('Downloading malwaredomains db from %s', url)\n    urlretrieve(url, path)\n\n\n@sopel.module.rule(r'(?u).*(https?://\\S+).*')\n@sopel.module.priority('high')\ndef url_handler(bot, trigger):\n    \"\"\"Checks for malicious URLs\"\"\"\n    check = True    # Enable URL checking\n    strict = False  # Strict mode: kick on malicious URL\n    positives = 0   # Number of engines saying it's malicious\n    total = 0       # Number of total engines\n    use_vt = True   # Use VirusTotal\n    check = bot.config.safety.enabled_by_default\n    if check is None:\n        # If not set, assume default\n        check = True\n    # DB overrides config:\n    setting = bot.db.get_channel_value(trigger.sender, 'safety')\n    if setting is not None:\n        if setting == 'off':\n            return  # Not checking\n        elif setting in ['on', 'strict', 'local', 'local strict']:\n            check = True\n        if setting == 'strict' or setting == 'local strict':\n            strict = True\n        if setting == 'local' or setting == 'local strict':\n            use_vt = False\n\n    if not check:\n        return  # Not overridden by DB, configured default off\n\n    try:\n        netloc = urlparse(trigger.group(1)).netloc\n    except ValueError:\n        return  # Invalid IPv6 URL\n\n    if any(regex.search(netloc) for regex in known_good):\n        return  # Whitelisted\n\n    apikey = bot.config.safety.vt_api_key\n    try:\n        if apikey is not None and use_vt:\n            payload = {'resource': unicode(trigger),\n                       'apikey': apikey,\n                       'scan': '1'}\n\n            if trigger not in bot.memory['safety_cache']:\n                r = requests.post(vt_base_api_url + 'report', data=payload)\n                r.raise_for_status()\n                result = r.json()\n                fetched = time.time()\n                if all(k in result for k in ['positives', 'total']):\n                    # cache result only if it contains a scan report\n                    # TODO: handle checking back for results from queued scans\n                    data = {'positives': result['positives'],\n                            'total': result['total'],\n                            'fetched': fetched}\n                    bot.memory['safety_cache'][trigger] = data\n                    if len(bot.memory['safety_cache']) >= (2 * cache_limit):\n                        _clean_cache(bot)\n            else:\n                print('using cache')\n                result = bot.memory['safety_cache'][trigger]\n            positives = result.get('positives', 0)\n            total = result.get('total', 0)\n    except requests.exceptions.RequestException:\n        # Ignoring exceptions with VT so MalwareDomains will always work\n        LOGGER.debug('[VirusTotal] Error obtaining response.', exc_info=True)\n    except InvalidJSONResponse:\n        # Ignoring exceptions with VT so MalwareDomains will always work\n        LOGGER.debug('[VirusTotal] Malformed response (invalid JSON).', exc_info=True)\n\n    if unicode(netloc).lower() in malware_domains:\n        # malwaredomains is more trustworthy than some VT engines\n        # therefore it gets a weight of 10 engines when calculating confidence\n        positives += 10\n        total += 10\n\n    if positives > 1:\n        # Possibly malicious URL detected!\n        confidence = '{}%'.format(round((positives / total) * 100))\n        msg = 'link posted by %s is possibly malicious ' % bold(trigger.nick)\n        msg += '(confidence %s - %s/%s)' % (confidence, positives, total)\n        bot.say('[' + bold(color('WARNING', 'red')) + '] ' + msg)\n        if strict:\n            bot.kick(trigger.nick, trigger.sender, 'Posted a malicious link')\n\n\n@sopel.module.commands('safety')\ndef toggle_safety(bot, trigger):\n    \"\"\"Set safety setting for channel\"\"\"\n    if not trigger.admin and bot.channels[trigger.sender].privileges[trigger.nick] < OP:\n        bot.reply('Only channel operators can change safety settings')\n        return\n    allowed_states = ['strict', 'on', 'off', 'local', 'local strict']\n    if not trigger.group(2) or trigger.group(2).lower() not in allowed_states:\n        options = ' / '.join(allowed_states)\n        bot.reply('Available options: %s' % options)\n        return\n\n    channel = trigger.sender.lower()\n    bot.db.set_channel_value(channel, 'safety', trigger.group(2).lower())\n    bot.reply('Safety is now set to \"%s\" on this channel' % trigger.group(2))\n\n\n# Clean the cache every day\n# Code above also calls this if there are too many cache entries\n@sopel.module.interval(24 * 60 * 60)\ndef _clean_cache(bot):\n    \"\"\"Cleans up old entries in URL safety cache.\"\"\"\n    if bot.memory['safety_cache_lock'].acquire(False):\n        LOGGER.info('Starting safety cache cleanup...')\n        try:\n            # clean up by age first\n            cutoff = time.time() - (7 * 24 * 60 * 60)  # 7 days ago\n            old_keys = []\n            for key, data in sopel.tools.iteritems(bot.memory['safety_cache']):\n                if data['fetched'] <= cutoff:\n                    old_keys.append(key)\n            for key in old_keys:\n                bot.memory['safety_cache'].pop(key, None)\n\n            # clean up more values if the cache is still too big\n            overage = len(bot.memory['safety_cache']) - cache_limit\n            if overage > 0:\n                extra_keys = sorted(\n                    (data.fetched, key)\n                    for (key, data)\n                    in bot.memory['safety_cache'].items())[:overage]\n                for (_, key) in extra_keys:\n                    bot.memory['safety_cache'].pop(key, None)\n        finally:\n            # No matter what errors happen (or not), release the lock\n            bot.memory['safety_cache_lock'].release()\n\n        LOGGER.info('Safety cache cleanup finished.')\n    else:\n        LOGGER.info(\n            'Skipping safety cache cleanup: Cache is locked, '\n            'cleanup already running.')\n", "path": "sopel/modules/safety.py"}]}