We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
safety: _clean_cache() doesn't, not really
Behold, the `safety` module's cache-cleaning function:

https://github.com/sopel-irc/sopel/blob/39e3680db18c4bed9801d040f7486c655b95a9a0/sopel/modules/safety.py#L207-L220

It's called once every 24 hours, and by `url_handler()` when the cache has too many entries. It seems to remove precisely _one_ cache entry—the oldest—every time it's called. [To quote @HumorBaby](https://github.com/sopel-irc/sopel/pull/1569#discussion_r278521202), "Not really a `_clean`'ing if you ask me :stuck_out_tongue_closed_eyes:"

Ideally, it would:

  * Remove any entries older than some reasonable threshold (a week?)
  * If there are still too many entries, continue removing the oldest one until below the limit (presently 1024)

Improvements on this algorithm are, as always, welcome. I'm just tossing out a hastily thrown-together idea for fixing this.

_Discovered in unrelated review: https://github.com/sopel-irc/sopel/pull/1569#discussion_r278354791_

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `sopel/modules/safety.py`
Content:
```
1 # coding=utf-8
2 """
3 safety.py - Alerts about malicious URLs
4 Copyright © 2014, Elad Alfassa, <elad@fedoraproject.org>
5 Licensed under the Eiffel Forum License 2.
6 
7 This module uses virustotal.com
8 """
9 from __future__ import unicode_literals, absolute_import, print_function, division
10 
11 import logging
12 import os.path
13 import re
14 import sys
15 import time
16 
17 import requests
18 
19 from sopel.config.types import StaticSection, ValidatedAttribute, ListAttribute
20 from sopel.formatting import color, bold
21 from sopel.module import OP
22 import sopel.tools
23 
24 try:
25     # This is done separately from the below version if/else because JSONDecodeError
26     # didn't appear until Python 3.5, but Sopel claims support for 3.3+
27     # Redo this whole block of nonsense when dropping py2/old py3 support
28     from json import JSONDecodeError as InvalidJSONResponse
29 except ImportError:
30     InvalidJSONResponse = ValueError
31 
32 if sys.version_info.major > 2:
33     unicode = str
34     from urllib.request import urlretrieve
35     from urllib.parse import urlparse
36 else:
37     from urllib import urlretrieve
38     from urlparse import urlparse
39 
40 
41 LOGGER = logging.getLogger(__name__)
42 
43 vt_base_api_url = 'https://www.virustotal.com/vtapi/v2/url/'
44 malware_domains = set()
45 known_good = []
46 
47 
48 class SafetySection(StaticSection):
49     enabled_by_default = ValidatedAttribute('enabled_by_default', bool, default=True)
50     """Whether to enable URL safety in all channels where it isn't explicitly disabled."""
51     known_good = ListAttribute('known_good')
52     """List of "known good" domains to ignore."""
53     vt_api_key = ValidatedAttribute('vt_api_key')
54     """Optional VirusTotal API key (improves malicious URL detection)."""
55 
56 
57 def configure(config):
58     """
59     | name | example | purpose |
60     | ---- | ------- | ------- |
61     | enabled\\_by\\_default | True | Enable URL safety in all channels where it isn't explicitly disabled. |
62     | known\\_good | sopel.chat,dftba.net | List of "known good" domains to ignore. |
63     | vt\\_api\\_key | 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef | Optional VirusTotal API key to improve malicious URL detection |
64     """
65     config.define_section('safety', SafetySection)
66     config.safety.configure_setting(
67         'enabled_by_default',
68         "Enable URL safety in channels that don't specifically disable it?",
69     )
70     config.safety.configure_setting(
71         'known_good',
72         'Enter any domains to whitelist',
73     )
74     config.safety.configure_setting(
75         'vt_api_key',
76         "Optionally, enter a VirusTotal API key to improve malicious URL "
77         "protection.\nOtherwise, only the Malwarebytes DB will be used."
78     )
79 
80 
81 def setup(bot):
82     bot.config.define_section('safety', SafetySection)
83 
84     if 'safety_cache' not in bot.memory:
85         bot.memory['safety_cache'] = sopel.tools.SopelMemory()
86     for item in bot.config.safety.known_good:
87         known_good.append(re.compile(item, re.I))
88 
89     loc = os.path.join(bot.config.homedir, 'malwaredomains.txt')
90     if os.path.isfile(loc):
91         if os.path.getmtime(loc) < time.time() - 24 * 60 * 60 * 7:
92             # File exists but older than one week — update it
93             _download_malwaredomains_db(loc)
94     else:
95         _download_malwaredomains_db(loc)
96     with open(loc, 'r') as f:
97         for line in f:
98             clean_line = unicode(line).strip().lower()
99             if clean_line != '':
100                 malware_domains.add(clean_line)
101 
102 
103 def shutdown(bot):
104     try:
105         del bot.memory['safety_cache']
106     except KeyError:
107         pass
108 
109 
110 def _download_malwaredomains_db(path):
111     url = 'https://mirror1.malwaredomains.com/files/justdomains'
112     LOGGER.info('Downloading malwaredomains db from %s', url)
113     urlretrieve(url, path)
114 
115 
116 @sopel.module.rule(r'(?u).*(https?://\S+).*')
117 @sopel.module.priority('high')
118 def url_handler(bot, trigger):
119     """Checks for malicious URLs"""
120     check = True    # Enable URL checking
121     strict = False  # Strict mode: kick on malicious URL
122     positives = 0   # Number of engines saying it's malicious
123     total = 0       # Number of total engines
124     use_vt = True   # Use VirusTotal
125     check = bot.config.safety.enabled_by_default
126     if check is None:
127         # If not set, assume default
128         check = True
129     # DB overrides config:
130     setting = bot.db.get_channel_value(trigger.sender, 'safety')
131     if setting is not None:
132         if setting == 'off':
133             return  # Not checking
134         elif setting in ['on', 'strict', 'local', 'local strict']:
135             check = True
136         if setting == 'strict' or setting == 'local strict':
137             strict = True
138         if setting == 'local' or setting == 'local strict':
139             use_vt = False
140 
141     if not check:
142         return  # Not overridden by DB, configured default off
143 
144     try:
145         netloc = urlparse(trigger.group(1)).netloc
146     except ValueError:
147         return  # Invalid IPv6 URL
148 
149     if any(regex.search(netloc) for regex in known_good):
150         return  # Whitelisted
151 
152     apikey = bot.config.safety.vt_api_key
153     try:
154         if apikey is not None and use_vt:
155             payload = {'resource': unicode(trigger),
156                        'apikey': apikey,
157                        'scan': '1'}
158 
159             if trigger not in bot.memory['safety_cache']:
160                 r = requests.post(vt_base_api_url + 'report', data=payload)
161                 r.raise_for_status()
162                 result = r.json()
163                 age = time.time()
164                 data = {'positives': result['positives'],
165                         'total': result['total'],
166                         'age': age}
167                 bot.memory['safety_cache'][trigger] = data
168                 if len(bot.memory['safety_cache']) > 1024:
169                     _clean_cache(bot)
170             else:
171                 print('using cache')
172                 result = bot.memory['safety_cache'][trigger]
173             positives = result['positives']
174             total = result['total']
175     except requests.exceptions.RequestException:
176         # Ignoring exceptions with VT so MalwareDomains will always work
177         LOGGER.debug('[VirusTotal] Error obtaining response.', exc_info=True)
178     except InvalidJSONResponse:
179         # Ignoring exceptions with VT so MalwareDomains will always work
180         LOGGER.debug('[VirusTotal] Malformed response (invalid JSON).', exc_info=True)
181 
182     if unicode(netloc).lower() in malware_domains:
183         # malwaredomains is more trustworthy than some VT engines
184         # therefore it gets a weight of 10 engines when calculating confidence
185         positives += 10
186         total += 10
187 
188     if positives > 1:
189         # Possibly malicious URL detected!
190         confidence = '{}%'.format(round((positives / total) * 100))
191         msg = 'link posted by %s is possibly malicious ' % bold(trigger.nick)
192         msg += '(confidence %s - %s/%s)' % (confidence, positives, total)
193         bot.say('[' + bold(color('WARNING', 'red')) + '] ' + msg)
194         if strict:
195             bot.kick(trigger.nick, trigger.sender, 'Posted a malicious link')
196 
197 
198 @sopel.module.commands('safety')
199 def toggle_safety(bot, trigger):
200     """Set safety setting for channel"""
201     if not trigger.admin and bot.channels[trigger.sender].privileges[trigger.nick] < OP:
202         bot.reply('Only channel operators can change safety settings')
203         return
204     allowed_states = ['strict', 'on', 'off', 'local', 'local strict']
205     if not trigger.group(2) or trigger.group(2).lower() not in allowed_states:
206         options = ' / '.join(allowed_states)
207         bot.reply('Available options: %s' % options)
208         return
209 
210     channel = trigger.sender.lower()
211     bot.db.set_channel_value(channel, 'safety', trigger.group(2).lower())
212     bot.reply('Safety is now set to "%s" on this channel' % trigger.group(2))
213 
214 
215 # Clean the cache every day
216 # Code above also calls this if there are too many cache entries
217 @sopel.module.interval(24 * 60 * 60)
218 def _clean_cache(bot):
219     """Cleans up old entries in URL cache"""
220     # TODO: probably should use locks here, to make sure stuff doesn't explode
221     oldest_key_age = 0
222     oldest_key = ''
223     for key, data in sopel.tools.iteritems(bot.memory['safety_cache']):
224         if data['age'] > oldest_key_age:
225             oldest_key_age = data['age']
226             oldest_key = key
227     if oldest_key in bot.memory['safety_cache']:
228         del bot.memory['safety_cache'][oldest_key]
229
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/sopel/modules/safety.py b/sopel/modules/safety.py
--- a/sopel/modules/safety.py
+++ b/sopel/modules/safety.py
@@ -12,6 +12,7 @@
 import os.path
 import re
 import sys
+import threading
 import time
 
 import requests
@@ -43,6 +44,7 @@
 vt_base_api_url = 'https://www.virustotal.com/vtapi/v2/url/'
 malware_domains = set()
 known_good = []
+cache_limit = 512
 
 
 class SafetySection(StaticSection):
@@ -83,6 +85,8 @@
 
     if 'safety_cache' not in bot.memory:
         bot.memory['safety_cache'] = sopel.tools.SopelMemory()
+    if 'safety_cache_lock' not in bot.memory:
+        bot.memory['safety_cache_lock'] = threading.Lock()
     for item in bot.config.safety.known_good:
         known_good.append(re.compile(item, re.I))
 
@@ -101,10 +105,8 @@
 
 
 def shutdown(bot):
-    try:
-        del bot.memory['safety_cache']
-    except KeyError:
-        pass
+    bot.memory.pop('safety_cache', None)
+    bot.memory.pop('safety_cache_lock', None)
 
 
 def _download_malwaredomains_db(path):
@@ -160,12 +162,12 @@
                 r = requests.post(vt_base_api_url + 'report', data=payload)
                 r.raise_for_status()
                 result = r.json()
-                age = time.time()
+                fetched = time.time()
                 data = {'positives': result['positives'],
                         'total': result['total'],
-                        'age': age}
+                        'fetched': fetched}
                 bot.memory['safety_cache'][trigger] = data
-                if len(bot.memory['safety_cache']) > 1024:
+                if len(bot.memory['safety_cache']) >= (2 * cache_limit):
                     _clean_cache(bot)
             else:
                 print('using cache')
@@ -216,13 +218,34 @@
 # Code above also calls this if there are too many cache entries
 @sopel.module.interval(24 * 60 * 60)
 def _clean_cache(bot):
-    """Cleans up old entries in URL cache"""
-    # TODO: probably should use locks here, to make sure stuff doesn't explode
-    oldest_key_age = 0
-    oldest_key = ''
-    for key, data in sopel.tools.iteritems(bot.memory['safety_cache']):
-        if data['age'] > oldest_key_age:
-            oldest_key_age = data['age']
-            oldest_key = key
-    if oldest_key in bot.memory['safety_cache']:
-        del bot.memory['safety_cache'][oldest_key]
+    """Cleans up old entries in URL safety cache."""
+    if bot.memory['safety_cache_lock'].acquire(False):
+        LOGGER.info('Starting safety cache cleanup...')
+        try:
+            # clean up by age first
+            cutoff = time.time() - (7 * 24 * 60 * 60)  # 7 days ago
+            old_keys = []
+            for key, data in sopel.tools.iteritems(bot.memory['safety_cache']):
+                if data['fetched'] <= cutoff:
+                    old_keys.append(key)
+            for key in old_keys:
+                bot.memory['safety_cache'].pop(key, None)
+
+            # clean up more values if the cache is still too big
+            overage = bot.memory['safety_cache'] - cache_limit
+            if overage > 0:
+                extra_keys = sorted(
+                    (data.fetched, key)
+                    for (key, data)
+                    in bot.memory['safety_cache'].items())[:overage]
+                for (_, key) in extra_keys:
+                    bot.memory['safety_cache'].pop(key, None)
+        finally:
+            # No matter what errors happen (or not), release the lock
+            bot.memory['safety_cache_lock'].release()
+
+        LOGGER.info('Safety cache cleanup finished.')
+    else:
+        LOGGER.info(
+            'Skipping safety cache cleanup: Cache is locked, '
+            'cleanup already running.')

{"golden_diff": "diff --git a/sopel/modules/safety.py b/sopel/modules/safety.py\n--- a/sopel/modules/safety.py\n+++ b/sopel/modules/safety.py\n@@ -12,6 +12,7 @@\n import os.path\n import re\n import sys\n+import threading\n import time\n \n import requests\n@@ -43,6 +44,7 @@\n vt_base_api_url = 'https://www.virustotal.com/vtapi/v2/url/'\n malware_domains = set()\n known_good = []\n+cache_limit = 512\n \n \n class SafetySection(StaticSection):\n@@ -83,6 +85,8 @@\n \n     if 'safety_cache' not in bot.memory:\n         bot.memory['safety_cache'] = sopel.tools.SopelMemory()\n+    if 'safety_cache_lock' not in bot.memory:\n+        bot.memory['safety_cache_lock'] = threading.Lock()\n     for item in bot.config.safety.known_good:\n         known_good.append(re.compile(item, re.I))\n \n@@ -101,10 +105,8 @@\n \n \n def shutdown(bot):\n-    try:\n-        del bot.memory['safety_cache']\n-    except KeyError:\n-        pass\n+    bot.memory.pop('safety_cache', None)\n+    bot.memory.pop('safety_cache_lock', None)\n \n \n def _download_malwaredomains_db(path):\n@@ -160,12 +162,12 @@\n                 r = requests.post(vt_base_api_url + 'report', data=payload)\n                 r.raise_for_status()\n                 result = r.json()\n-                age = time.time()\n+                fetched = time.time()\n                 data = {'positives': result['positives'],\n                         'total': result['total'],\n-                        'age': age}\n+                        'fetched': fetched}\n                 bot.memory['safety_cache'][trigger] = data\n-                if len(bot.memory['safety_cache']) > 1024:\n+                if len(bot.memory['safety_cache']) >= (2 * cache_limit):\n                     _clean_cache(bot)\n             else:\n                 print('using cache')\n@@ -216,13 +218,34 @@\n # Code above also calls this if there are too many cache entries\n @sopel.module.interval(24 * 60 * 60)\n def _clean_cache(bot):\n-    \"\"\"Cleans up old entries in URL cache\"\"\"\n-    # TODO: probably should use locks here, to make sure stuff doesn't explode\n-    oldest_key_age = 0\n-    oldest_key = ''\n-    for key, data in sopel.tools.iteritems(bot.memory['safety_cache']):\n-        if data['age'] > oldest_key_age:\n-            oldest_key_age = data['age']\n-            oldest_key = key\n-    if oldest_key in bot.memory['safety_cache']:\n-        del bot.memory['safety_cache'][oldest_key]\n+    \"\"\"Cleans up old entries in URL safety cache.\"\"\"\n+    if bot.memory['safety_cache_lock'].acquire(False):\n+        LOGGER.info('Starting safety cache cleanup...')\n+        try:\n+            # clean up by age first\n+            cutoff = time.time() - (7 * 24 * 60 * 60)  # 7 days ago\n+            old_keys = []\n+            for key, data in sopel.tools.iteritems(bot.memory['safety_cache']):\n+                if data['fetched'] <= cutoff:\n+                    old_keys.append(key)\n+            for key in old_keys:\n+                bot.memory['safety_cache'].pop(key, None)\n+\n+            # clean up more values if the cache is still too big\n+            overage = bot.memory['safety_cache'] - cache_limit\n+            if overage > 0:\n+                extra_keys = sorted(\n+                    (data.fetched, key)\n+                    for (key, data)\n+                    in bot.memory['safety_cache'].items())[:overage]\n+                for (_, key) in extra_keys:\n+                    bot.memory['safety_cache'].pop(key, None)\n+        finally:\n+            # No matter what errors happen (or not), release the lock\n+            bot.memory['safety_cache_lock'].release()\n+\n+        LOGGER.info('Safety cache cleanup finished.')\n+    else:\n+        LOGGER.info(\n+            'Skipping safety cache cleanup: Cache is locked, '\n+            'cleanup already running.')\n", "issue": "safety: _clean_cache() doesn't, not really\nBehold, the `safety` module's cache-cleaning function:\r\n\r\nhttps://github.com/sopel-irc/sopel/blob/39e3680db18c4bed9801d040f7486c655b95a9a0/sopel/modules/safety.py#L207-L220\r\n\r\nIt's called once every 24 hours, and by `url_handler()` when the cache has too many entries. It seems to remove precisely _one_ cache entry\u2014the oldest\u2014every time it's called. [To quote @HumorBaby](https://github.com/sopel-irc/sopel/pull/1569#discussion_r278521202), \"Not really a `_clean`'ing if you ask me :stuck_out_tongue_closed_eyes:\"\r\n\r\nIdeally, it would:\r\n\r\n  * Remove any entries older than some reasonable threshold (a week?)\r\n  * If there are still too many entries, continue removing the oldest one until below the limit (presently 1024)\r\n\r\nImprovements on this algorithm are, as always, welcome. I'm just tossing out a hastily thrown-together idea for fixing this.\r\n\r\n_Discovered in unrelated review: https://github.com/sopel-irc/sopel/pull/1569#discussion_r278354791_\n", "before_files": [{"content": "# coding=utf-8\n\"\"\"\nsafety.py - Alerts about malicious URLs\nCopyright \u00a9 2014, Elad Alfassa, <elad@fedoraproject.org>\nLicensed under the Eiffel Forum License 2.\n\nThis module uses virustotal.com\n\"\"\"\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport logging\nimport os.path\nimport re\nimport sys\nimport time\n\nimport requests\n\nfrom sopel.config.types import StaticSection, ValidatedAttribute, ListAttribute\nfrom sopel.formatting import color, bold\nfrom sopel.module import OP\nimport sopel.tools\n\ntry:\n    # This is done separately from the below version if/else because JSONDecodeError\n    # didn't appear until Python 3.5, but Sopel claims support for 3.3+\n    # Redo this whole block of nonsense when dropping py2/old py3 support\n    from json import JSONDecodeError as InvalidJSONResponse\nexcept ImportError:\n    InvalidJSONResponse = ValueError\n\nif sys.version_info.major > 2:\n    unicode = str\n    from urllib.request import urlretrieve\n    from urllib.parse import urlparse\nelse:\n    from urllib import urlretrieve\n    from urlparse import urlparse\n\n\nLOGGER = logging.getLogger(__name__)\n\nvt_base_api_url = 'https://www.virustotal.com/vtapi/v2/url/'\nmalware_domains = set()\nknown_good = []\n\n\nclass SafetySection(StaticSection):\n    enabled_by_default = ValidatedAttribute('enabled_by_default', bool, default=True)\n    \"\"\"Whether to enable URL safety in all channels where it isn't explicitly disabled.\"\"\"\n    known_good = ListAttribute('known_good')\n    \"\"\"List of \"known good\" domains to ignore.\"\"\"\n    vt_api_key = ValidatedAttribute('vt_api_key')\n    \"\"\"Optional VirusTotal API key (improves malicious URL detection).\"\"\"\n\n\ndef configure(config):\n    \"\"\"\n    | name | example | purpose |\n    | ---- | ------- | ------- |\n    | enabled\\\\_by\\\\_default | True | Enable URL safety in all channels where it isn't explicitly disabled. |\n    | known\\\\_good | sopel.chat,dftba.net | List of \"known good\" domains to ignore. |\n    | vt\\\\_api\\\\_key | 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef | Optional VirusTotal API key to improve malicious URL detection |\n    \"\"\"\n    config.define_section('safety', SafetySection)\n    config.safety.configure_setting(\n        'enabled_by_default',\n        \"Enable URL safety in channels that don't specifically disable it?\",\n    )\n    config.safety.configure_setting(\n        'known_good',\n        'Enter any domains to whitelist',\n    )\n    config.safety.configure_setting(\n        'vt_api_key',\n        \"Optionally, enter a VirusTotal API key to improve malicious URL \"\n        \"protection.\\nOtherwise, only the Malwarebytes DB will be used.\"\n    )\n\n\ndef setup(bot):\n    bot.config.define_section('safety', SafetySection)\n\n    if 'safety_cache' not in bot.memory:\n        bot.memory['safety_cache'] = sopel.tools.SopelMemory()\n    for item in bot.config.safety.known_good:\n        known_good.append(re.compile(item, re.I))\n\n    loc = os.path.join(bot.config.homedir, 'malwaredomains.txt')\n    if os.path.isfile(loc):\n        if os.path.getmtime(loc) < time.time() - 24 * 60 * 60 * 7:\n            # File exists but older than one week \u2014 update it\n            _download_malwaredomains_db(loc)\n    else:\n        _download_malwaredomains_db(loc)\n    with open(loc, 'r') as f:\n        for line in f:\n            clean_line = unicode(line).strip().lower()\n            if clean_line != '':\n                malware_domains.add(clean_line)\n\n\ndef shutdown(bot):\n    try:\n        del bot.memory['safety_cache']\n    except KeyError:\n        pass\n\n\ndef _download_malwaredomains_db(path):\n    url = 'https://mirror1.malwaredomains.com/files/justdomains'\n    LOGGER.info('Downloading malwaredomains db from %s', url)\n    urlretrieve(url, path)\n\n\n@sopel.module.rule(r'(?u).*(https?://\\S+).*')\n@sopel.module.priority('high')\ndef url_handler(bot, trigger):\n    \"\"\"Checks for malicious URLs\"\"\"\n    check = True    # Enable URL checking\n    strict = False  # Strict mode: kick on malicious URL\n    positives = 0   # Number of engines saying it's malicious\n    total = 0       # Number of total engines\n    use_vt = True   # Use VirusTotal\n    check = bot.config.safety.enabled_by_default\n    if check is None:\n        # If not set, assume default\n        check = True\n    # DB overrides config:\n    setting = bot.db.get_channel_value(trigger.sender, 'safety')\n    if setting is not None:\n        if setting == 'off':\n            return  # Not checking\n        elif setting in ['on', 'strict', 'local', 'local strict']:\n            check = True\n        if setting == 'strict' or setting == 'local strict':\n            strict = True\n        if setting == 'local' or setting == 'local strict':\n            use_vt = False\n\n    if not check:\n        return  # Not overridden by DB, configured default off\n\n    try:\n        netloc = urlparse(trigger.group(1)).netloc\n    except ValueError:\n        return  # Invalid IPv6 URL\n\n    if any(regex.search(netloc) for regex in known_good):\n        return  # Whitelisted\n\n    apikey = bot.config.safety.vt_api_key\n    try:\n        if apikey is not None and use_vt:\n            payload = {'resource': unicode(trigger),\n                       'apikey': apikey,\n                       'scan': '1'}\n\n            if trigger not in bot.memory['safety_cache']:\n                r = requests.post(vt_base_api_url + 'report', data=payload)\n                r.raise_for_status()\n                result = r.json()\n                age = time.time()\n                data = {'positives': result['positives'],\n                        'total': result['total'],\n                        'age': age}\n                bot.memory['safety_cache'][trigger] = data\n                if len(bot.memory['safety_cache']) > 1024:\n                    _clean_cache(bot)\n            else:\n                print('using cache')\n                result = bot.memory['safety_cache'][trigger]\n            positives = result['positives']\n            total = result['total']\n    except requests.exceptions.RequestException:\n        # Ignoring exceptions with VT so MalwareDomains will always work\n        LOGGER.debug('[VirusTotal] Error obtaining response.', exc_info=True)\n    except InvalidJSONResponse:\n        # Ignoring exceptions with VT so MalwareDomains will always work\n        LOGGER.debug('[VirusTotal] Malformed response (invalid JSON).', exc_info=True)\n\n    if unicode(netloc).lower() in malware_domains:\n        # malwaredomains is more trustworthy than some VT engines\n        # therefore it gets a weight of 10 engines when calculating confidence\n        positives += 10\n        total += 10\n\n    if positives > 1:\n        # Possibly malicious URL detected!\n        confidence = '{}%'.format(round((positives / total) * 100))\n        msg = 'link posted by %s is possibly malicious ' % bold(trigger.nick)\n        msg += '(confidence %s - %s/%s)' % (confidence, positives, total)\n        bot.say('[' + bold(color('WARNING', 'red')) + '] ' + msg)\n        if strict:\n            bot.kick(trigger.nick, trigger.sender, 'Posted a malicious link')\n\n\n@sopel.module.commands('safety')\ndef toggle_safety(bot, trigger):\n    \"\"\"Set safety setting for channel\"\"\"\n    if not trigger.admin and bot.channels[trigger.sender].privileges[trigger.nick] < OP:\n        bot.reply('Only channel operators can change safety settings')\n        return\n    allowed_states = ['strict', 'on', 'off', 'local', 'local strict']\n    if not trigger.group(2) or trigger.group(2).lower() not in allowed_states:\n        options = ' / '.join(allowed_states)\n        bot.reply('Available options: %s' % options)\n        return\n\n    channel = trigger.sender.lower()\n    bot.db.set_channel_value(channel, 'safety', trigger.group(2).lower())\n    bot.reply('Safety is now set to \"%s\" on this channel' % trigger.group(2))\n\n\n# Clean the cache every day\n# Code above also calls this if there are too many cache entries\n@sopel.module.interval(24 * 60 * 60)\ndef _clean_cache(bot):\n    \"\"\"Cleans up old entries in URL cache\"\"\"\n    # TODO: probably should use locks here, to make sure stuff doesn't explode\n    oldest_key_age = 0\n    oldest_key = ''\n    for key, data in sopel.tools.iteritems(bot.memory['safety_cache']):\n        if data['age'] > oldest_key_age:\n            oldest_key_age = data['age']\n            oldest_key = key\n    if oldest_key in bot.memory['safety_cache']:\n        del bot.memory['safety_cache'][oldest_key]\n", "path": "sopel/modules/safety.py"}], "after_files": [{"content": "# coding=utf-8\n\"\"\"\nsafety.py - Alerts about malicious URLs\nCopyright \u00a9 2014, Elad Alfassa, <elad@fedoraproject.org>\nLicensed under the Eiffel Forum License 2.\n\nThis module uses virustotal.com\n\"\"\"\nfrom __future__ import unicode_literals, absolute_import, print_function, division\n\nimport logging\nimport os.path\nimport re\nimport sys\nimport threading\nimport time\n\nimport requests\n\nfrom sopel.config.types import StaticSection, ValidatedAttribute, ListAttribute\nfrom sopel.formatting import color, bold\nfrom sopel.module import OP\nimport sopel.tools\n\ntry:\n    # This is done separately from the below version if/else because JSONDecodeError\n    # didn't appear until Python 3.5, but Sopel claims support for 3.3+\n    # Redo this whole block of nonsense when dropping py2/old py3 support\n    from json import JSONDecodeError as InvalidJSONResponse\nexcept ImportError:\n    InvalidJSONResponse = ValueError\n\nif sys.version_info.major > 2:\n    unicode = str\n    from urllib.request import urlretrieve\n    from urllib.parse import urlparse\nelse:\n    from urllib import urlretrieve\n    from urlparse import urlparse\n\n\nLOGGER = logging.getLogger(__name__)\n\nvt_base_api_url = 'https://www.virustotal.com/vtapi/v2/url/'\nmalware_domains = set()\nknown_good = []\ncache_limit = 512\n\n\nclass SafetySection(StaticSection):\n    enabled_by_default = ValidatedAttribute('enabled_by_default', bool, default=True)\n    \"\"\"Whether to enable URL safety in all channels where it isn't explicitly disabled.\"\"\"\n    known_good = ListAttribute('known_good')\n    \"\"\"List of \"known good\" domains to ignore.\"\"\"\n    vt_api_key = ValidatedAttribute('vt_api_key')\n    \"\"\"Optional VirusTotal API key (improves malicious URL detection).\"\"\"\n\n\ndef configure(config):\n    \"\"\"\n    | name | example | purpose |\n    | ---- | ------- | ------- |\n    | enabled\\\\_by\\\\_default | True | Enable URL safety in all channels where it isn't explicitly disabled. |\n    | known\\\\_good | sopel.chat,dftba.net | List of \"known good\" domains to ignore. |\n    | vt\\\\_api\\\\_key | 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef | Optional VirusTotal API key to improve malicious URL detection |\n    \"\"\"\n    config.define_section('safety', SafetySection)\n    config.safety.configure_setting(\n        'enabled_by_default',\n        \"Enable URL safety in channels that don't specifically disable it?\",\n    )\n    config.safety.configure_setting(\n        'known_good',\n        'Enter any domains to whitelist',\n    )\n    config.safety.configure_setting(\n        'vt_api_key',\n        \"Optionally, enter a VirusTotal API key to improve malicious URL \"\n        \"protection.\\nOtherwise, only the Malwarebytes DB will be used.\"\n    )\n\n\ndef setup(bot):\n    bot.config.define_section('safety', SafetySection)\n\n    if 'safety_cache' not in bot.memory:\n        bot.memory['safety_cache'] = sopel.tools.SopelMemory()\n    if 'safety_cache_lock' not in bot.memory:\n        bot.memory['safety_cache_lock'] = threading.Lock()\n    for item in bot.config.safety.known_good:\n        known_good.append(re.compile(item, re.I))\n\n    loc = os.path.join(bot.config.homedir, 'malwaredomains.txt')\n    if os.path.isfile(loc):\n        if os.path.getmtime(loc) < time.time() - 24 * 60 * 60 * 7:\n            # File exists but older than one week \u2014 update it\n            _download_malwaredomains_db(loc)\n    else:\n        _download_malwaredomains_db(loc)\n    with open(loc, 'r') as f:\n        for line in f:\n            clean_line = unicode(line).strip().lower()\n            if clean_line != '':\n                malware_domains.add(clean_line)\n\n\ndef shutdown(bot):\n    bot.memory.pop('safety_cache', None)\n    bot.memory.pop('safety_cache_lock', None)\n\n\ndef _download_malwaredomains_db(path):\n    url = 'https://mirror1.malwaredomains.com/files/justdomains'\n    LOGGER.info('Downloading malwaredomains db from %s', url)\n    urlretrieve(url, path)\n\n\n@sopel.module.rule(r'(?u).*(https?://\\S+).*')\n@sopel.module.priority('high')\ndef url_handler(bot, trigger):\n    \"\"\"Checks for malicious URLs\"\"\"\n    check = True    # Enable URL checking\n    strict = False  # Strict mode: kick on malicious URL\n    positives = 0   # Number of engines saying it's malicious\n    total = 0       # Number of total engines\n    use_vt = True   # Use VirusTotal\n    check = bot.config.safety.enabled_by_default\n    if check is None:\n        # If not set, assume default\n        check = True\n    # DB overrides config:\n    setting = bot.db.get_channel_value(trigger.sender, 'safety')\n    if setting is not None:\n        if setting == 'off':\n            return  # Not checking\n        elif setting in ['on', 'strict', 'local', 'local strict']:\n            check = True\n        if setting == 'strict' or setting == 'local strict':\n            strict = True\n        if setting == 'local' or setting == 'local strict':\n            use_vt = False\n\n    if not check:\n        return  # Not overridden by DB, configured default off\n\n    try:\n        netloc = urlparse(trigger.group(1)).netloc\n    except ValueError:\n        return  # Invalid IPv6 URL\n\n    if any(regex.search(netloc) for regex in known_good):\n        return  # Whitelisted\n\n    apikey = bot.config.safety.vt_api_key\n    try:\n        if apikey is not None and use_vt:\n            payload = {'resource': unicode(trigger),\n                       'apikey': apikey,\n                       'scan': '1'}\n\n            if trigger not in bot.memory['safety_cache']:\n                r = requests.post(vt_base_api_url + 'report', data=payload)\n                r.raise_for_status()\n                result = r.json()\n                fetched = time.time()\n                data = {'positives': result['positives'],\n                        'total': result['total'],\n                        'fetched': fetched}\n                bot.memory['safety_cache'][trigger] = data\n                if len(bot.memory['safety_cache']) >= (2 * cache_limit):\n                    _clean_cache(bot)\n            else:\n                print('using cache')\n                result = bot.memory['safety_cache'][trigger]\n            positives = result['positives']\n            total = result['total']\n    except requests.exceptions.RequestException:\n        # Ignoring exceptions with VT so MalwareDomains will always work\n        LOGGER.debug('[VirusTotal] Error obtaining response.', exc_info=True)\n    except InvalidJSONResponse:\n        # Ignoring exceptions with VT so MalwareDomains will always work\n        LOGGER.debug('[VirusTotal] Malformed response (invalid JSON).', exc_info=True)\n\n    if unicode(netloc).lower() in malware_domains:\n        # malwaredomains is more trustworthy than some VT engines\n        # therefore it gets a weight of 10 engines when calculating confidence\n        positives += 10\n        total += 10\n\n    if positives > 1:\n        # Possibly malicious URL detected!\n        confidence = '{}%'.format(round((positives / total) * 100))\n        msg = 'link posted by %s is possibly malicious ' % bold(trigger.nick)\n        msg += '(confidence %s - %s/%s)' % (confidence, positives, total)\n        bot.say('[' + bold(color('WARNING', 'red')) + '] ' + msg)\n        if strict:\n            bot.kick(trigger.nick, trigger.sender, 'Posted a malicious link')\n\n\n@sopel.module.commands('safety')\ndef toggle_safety(bot, trigger):\n    \"\"\"Set safety setting for channel\"\"\"\n    if not trigger.admin and bot.channels[trigger.sender].privileges[trigger.nick] < OP:\n        bot.reply('Only channel operators can change safety settings')\n        return\n    allowed_states = ['strict', 'on', 'off', 'local', 'local strict']\n    if not trigger.group(2) or trigger.group(2).lower() not in allowed_states:\n        options = ' / '.join(allowed_states)\n        bot.reply('Available options: %s' % options)\n        return\n\n    channel = trigger.sender.lower()\n    bot.db.set_channel_value(channel, 'safety', trigger.group(2).lower())\n    bot.reply('Safety is now set to \"%s\" on this channel' % trigger.group(2))\n\n\n# Clean the cache every day\n# Code above also calls this if there are too many cache entries\n@sopel.module.interval(24 * 60 * 60)\ndef _clean_cache(bot):\n    \"\"\"Cleans up old entries in URL safety cache.\"\"\"\n    if bot.memory['safety_cache_lock'].acquire(False):\n        LOGGER.info('Starting safety cache cleanup...')\n        try:\n            # clean up by age first\n            cutoff = time.time() - (7 * 24 * 60 * 60)  # 7 days ago\n            old_keys = []\n            for key, data in sopel.tools.iteritems(bot.memory['safety_cache']):\n                if data['fetched'] <= cutoff:\n                    old_keys.append(key)\n            for key in old_keys:\n                bot.memory['safety_cache'].pop(key, None)\n\n            # clean up more values if the cache is still too big\n            overage = bot.memory['safety_cache'] - cache_limit\n            if overage > 0:\n                extra_keys = sorted(\n                    (data.fetched, key)\n                    for (key, data)\n                    in bot.memory['safety_cache'].items())[:overage]\n                for (_, key) in extra_keys:\n                    bot.memory['safety_cache'].pop(key, None)\n        finally:\n            # No matter what errors happen (or not), release the lock\n            bot.memory['safety_cache_lock'].release()\n\n        LOGGER.info('Safety cache cleanup finished.')\n    else:\n        LOGGER.info(\n            'Skipping safety cache cleanup: Cache is locked, '\n            'cleanup already running.')\n", "path": "sopel/modules/safety.py"}]}