We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
RedisCache fails to delete cached objects when f=None
For background, see https://groups.google.com/d/msg/web2py/1LoXXrPlIGE/bX2u3CJ2BQAJ.

According to the book, calling `cache.model(key, f=None)` should cause `key` to be deleted from the cache. However, in `gluon.contrib.redis_cache.RedisClient.__call__`, setting `f=None` only deletes the cached object if it has expired.

This causes a problem with `@cache.action` because it relies on the `f=None` deletion method to delete pages that were cached with an invalid HTTP status.

We should change the behavior of the Redis cache to match that of the other cache models (i.e., setting `f=None` should _always_ delete the key from the cache).

Also, for extra safety, perhaps `@cache.action` should delete pages with an invalid status via `cache_model.clear(r'^%s$' % cache_key)` rather than `cache_model(cache_key, None)`.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `gluon/contrib/redis_cache.py`
Content:
```
1 """
2 Developed by niphlod@gmail.com
3 Released under web2py license because includes gluon/cache.py source code
4 """
5 
6 try:
7     import cPickle as pickle
8 except:
9     import pickle
10 import time
11 import re
12 import logging
13 import thread
14 import random
15 from gluon import current
16 from gluon.cache import CacheAbstract
17 from gluon.contrib.redis_utils import acquire_lock, release_lock
18 from gluon.contrib.redis_utils import register_release_lock, RConnectionError
19 
20 logger = logging.getLogger("web2py.cache.redis")
21 
22 locker = thread.allocate_lock()
23 
24 
25 def RedisCache(redis_conn=None, debug=False, with_lock=False, fail_gracefully=False, db=None):
26     """
27     Usage example: put in models::
28 
29     First of all install Redis
30     Ubuntu :
31     sudo apt-get install redis-server
32     sudo pip install redis
33 
34     Then
35 
36         from gluon.contrib.redis_utils import RConn
37         rconn = RConn()
38         from gluon.contrib.redis_cache import RedisCache
39         cache.redis = RedisCache(redis_conn=rconn, debug=True, with_lock=True)
40 
41     Args:
42         redis_conn: a redis-like connection object
43         debug: if True adds to stats() the total_hits and misses
44         with_lock: sets the default locking mode for creating new keys.
45             By default is False (usualy when you choose Redis you do it
46             for performances reason)
47             When True, only one thread/process can set a value concurrently
48         fail_gracefully: if redis is unavailable, returns the value computing it
49             instead of raising an exception
50 
51     It can be used pretty much the same as cache.ram()
52     When you use cache.redis directly you can use :
53 
54         redis_key_and_var_name = cache.redis('redis_key_and_var_name', lambda or function,
55                                              time_expire=time.time(), with_lock=True)
56 
57     to enforce locking. The with_lock parameter overrides the one set in the
58     cache.redis instance creation
59 
60     cache.redis.stats()
61         returns a dictionary with statistics of Redis server
62         with one additional key ('w2p_keys') showing all keys currently set
63         from web2py with their TTL
64 
65     A little wording on how keys are stored (and why the cache_it() function
66     and the clear() one look a little bit convoluted): there are a lot of
67     libraries that just store values and then use the KEYS command to delete it.
68     Until recent releases of this module, that technique was used here too.
69     In the need of deleting specific keys in a database with zillions keys in it
70     (other web2py apps, other applications in the need of a Redis stack) the
71     KEYS command is slow (it needs to scan every key in the database).
72     So, we use Redis 'sets' to store keys in "buckets"...
73     - every key created gets "indexed" in a bucket
74     - all buckets are indexed in a fixed key that never expires
75     - all keys generated within the same minute go in the same bucket
76     - every bucket is then set to expire when every key within it is expired
77     When we need to clear() cached keys:
78     - we tell Redis to SUNION all buckets
79        - gives us just the keys that are not expired yet
80     - buckets that are expired are removed from the fixed set
81     - we scan the keys and then delete them
82     """
83 
84     locker.acquire()
85     try:
86         instance_name = 'redis_instance_' + current.request.application
87         if not hasattr(RedisCache, instance_name):
88             setattr(RedisCache, instance_name, 
89                     RedisClient(redis_conn=redis_conn, debug=debug,
90                                 with_lock=with_lock, fail_gracefully=fail_gracefully))
91         return getattr(RedisCache, instance_name)
92     finally:
93         locker.release()
94 
95 
96 class RedisClient(object):
97 
98     meta_storage = {}
99     MAX_RETRIES = 5
100     RETRIES = 0
101 
102     def __init__(self, redis_conn=None, debug=False,
103                  with_lock=False, fail_gracefully=False):
104         self.request = current.request
105         self.debug = debug
106         self.with_lock = with_lock
107         self.fail_gracefully = fail_gracefully
108         self.prefix = "w2p:cache:%s:" % self.request.application
109         if self.request:
110             app = self.request.application
111         else:
112             app = ''
113 
114         if app not in self.meta_storage:
115             self.storage = self.meta_storage[app] = {
116                 CacheAbstract.cache_stats_name: {
117                     'hit_total': 0,
118                     'misses': 0,
119                 }}
120         else:
121             self.storage = self.meta_storage[app]
122 
123         self.cache_set_key = 'w2p:%s:___cache_set' % self.request.application
124 
125         self.r_server = redis_conn
126         self._release_script = register_release_lock(self.r_server)
127 
128     def initialize(self):
129         pass
130 
131     def __call__(self, key, f, time_expire=300, with_lock=None):
132         if with_lock is None:
133             with_lock = self.with_lock
134         if time_expire is None:
135             time_expire = 24 * 60 * 60
136         newKey = self.__keyFormat__(key)
137         value = None
138         ttl = 0
139         try:
140             # is there a value
141             obj = self.r_server.get(newKey)
142             # what's its ttl
143             if obj:
144                 ttl = self.r_server.ttl(newKey)
145             if ttl > time_expire:
146                 obj = None
147             if obj:
148                 # was cached
149                 if self.debug:
150                     self.r_server.incr('web2py_cache_statistics:hit_total')
151                 value = pickle.loads(obj)
152             elif f is None:
153                 # delete and never look back
154                 self.r_server.delete(newKey)
155             else:
156                 # naive distributed locking
157                 if with_lock:
158                     lock_key = '%s:__lock' % newKey
159                     randomvalue = time.time()
160                     al = acquire_lock(self.r_server, lock_key, randomvalue)
161                     # someone may have computed it
162                     obj = self.r_server.get(newKey)
163                     if obj is None:
164                         value = self.cache_it(newKey, f, time_expire)
165                     else:
166                         value = pickle.loads(obj)
167                     release_lock(self, lock_key, al)
168                 else:
169                     # without distributed locking
170                     value = self.cache_it(newKey, f, time_expire)
171             return value
172         except RConnectionError:
173             return self.retry_call(key, f, time_expire, with_lock)
174 
175     def cache_it(self, key, f, time_expire):
176         if self.debug:
177             self.r_server.incr('web2py_cache_statistics:misses')
178         cache_set_key = self.cache_set_key
179         expire_at = int(time.time() + time_expire) + 120
180         bucket_key = "%s:%s" % (cache_set_key, expire_at / 60)
181         value = f()
182         value_ = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
183         if time_expire == 0:
184             time_expire = 1
185         self.r_server.setex(key, time_expire, value_)
186         # print '%s will expire on %s: it goes in bucket %s' % (key, time.ctime(expire_at))
187         # print 'that will expire on %s' % (bucket_key, time.ctime(((expire_at / 60) + 1) * 60))
188         p = self.r_server.pipeline()
189         # add bucket to the fixed set
190         p.sadd(cache_set_key, bucket_key)
191         # sets the key
192         p.setex(key, time_expire, value_)
193         # add the key to the bucket
194         p.sadd(bucket_key, key)
195         # expire the bucket properly
196         p.expireat(bucket_key, ((expire_at / 60) + 1) * 60)
197         p.execute()
198         return value
199 
200     def retry_call(self, key, f, time_expire, with_lock):
201         self.RETRIES += 1
202         if self.RETRIES <= self.MAX_RETRIES:
203             logger.error("sleeping %s seconds before reconnecting" % (2 * self.RETRIES))
204             time.sleep(2 * self.RETRIES)
205             if self.fail_gracefully:
206                 self.RETRIES = 0
207                 return f()
208             return self.__call__(key, f, time_expire, with_lock)
209         else:
210             self.RETRIES = 0
211             if self.fail_gracefully:
212                 return f
213             raise RConnectionError('Redis instance is unavailable')
214 
215     def increment(self, key, value=1):
216         try:
217             newKey = self.__keyFormat__(key)
218             return self.r_server.incr(newKey, value)
219         except RConnectionError:
220             return self.retry_increment(key, value)
221 
222     def retry_increment(self, key, value):
223         self.RETRIES += 1
224         if self.RETRIES <= self.MAX_RETRIES:
225             logger.error("sleeping some seconds before reconnecting")
226             time.sleep(2 * self.RETRIES)
227             return self.increment(key, value)
228         else:
229             self.RETRIES = 0
230             raise RConnectionError('Redis instance is unavailable')
231 
232     def clear(self, regex):
233         """
234         Auxiliary function called by `clear` to search and
235         clear cache entries
236         """
237         r = re.compile(regex)
238         # get all buckets
239         buckets = self.r_server.smembers(self.cache_set_key)
240         # get all keys in buckets
241         if buckets:
242             keys = self.r_server.sunion(buckets)
243         else:
244             return
245         prefix = self.prefix
246         pipe = self.r_server.pipeline()
247         for a in keys:
248             if r.match(str(a).replace(prefix, '', 1)):
249                 pipe.delete(a)
250         if random.randrange(0, 100) < 10:
251             # do this just once in a while (10% chance)
252             self.clear_buckets(buckets)
253         pipe.execute()
254 
255     def clear_buckets(self, buckets):
256         p = self.r_server.pipeline()
257         for b in buckets:
258             if not self.r_server.exists(b):
259                 p.srem(self.cache_set_key, b)
260         p.execute()
261 
262     def delete(self, key):
263         newKey = self.__keyFormat__(key)
264         return self.r_server.delete(newKey)
265 
266     def stats(self):
267         stats_collector = self.r_server.info()
268         if self.debug:
269             stats_collector['w2p_stats'] = dict(
270                 hit_total=self.r_server.get(
271                     'web2py_cache_statistics:hit_total'),
272                 misses=self.r_server.get('web2py_cache_statistics:misses')
273             )
274         stats_collector['w2p_keys'] = dict()
275 
276         for a in self.r_server.keys("w2p:%s:*" % (
277                 self.request.application)):
278             stats_collector['w2p_keys']["%s_expire_in_sec" % a] = self.r_server.ttl(a)
279         return stats_collector
280 
281     def __keyFormat__(self, key):
282         return '%s%s' % (self.prefix, key.replace(' ', '_'))
283
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/gluon/contrib/redis_cache.py b/gluon/contrib/redis_cache.py
--- a/gluon/contrib/redis_cache.py
+++ b/gluon/contrib/redis_cache.py
@@ -85,7 +85,7 @@
     try:
         instance_name = 'redis_instance_' + current.request.application
         if not hasattr(RedisCache, instance_name):
-            setattr(RedisCache, instance_name, 
+            setattr(RedisCache, instance_name,
                     RedisClient(redis_conn=redis_conn, debug=debug,
                                 with_lock=with_lock, fail_gracefully=fail_gracefully))
         return getattr(RedisCache, instance_name)
@@ -137,6 +137,10 @@
         value = None
         ttl = 0
         try:
+            if f is None:
+                # delete and never look back
+                self.r_server.delete(newKey)
+                return None
             # is there a value
             obj = self.r_server.get(newKey)
             # what's its ttl
@@ -149,9 +153,6 @@
                 if self.debug:
                     self.r_server.incr('web2py_cache_statistics:hit_total')
                 value = pickle.loads(obj)
-            elif f is None:
-                # delete and never look back
-                self.r_server.delete(newKey)
             else:
                 # naive distributed locking
                 if with_lock:

{"golden_diff": "diff --git a/gluon/contrib/redis_cache.py b/gluon/contrib/redis_cache.py\n--- a/gluon/contrib/redis_cache.py\n+++ b/gluon/contrib/redis_cache.py\n@@ -85,7 +85,7 @@\n     try:\n         instance_name = 'redis_instance_' + current.request.application\n         if not hasattr(RedisCache, instance_name):\n-            setattr(RedisCache, instance_name, \n+            setattr(RedisCache, instance_name,\n                     RedisClient(redis_conn=redis_conn, debug=debug,\n                                 with_lock=with_lock, fail_gracefully=fail_gracefully))\n         return getattr(RedisCache, instance_name)\n@@ -137,6 +137,10 @@\n         value = None\n         ttl = 0\n         try:\n+            if f is None:\n+                # delete and never look back\n+                self.r_server.delete(newKey)\n+                return None\n             # is there a value\n             obj = self.r_server.get(newKey)\n             # what's its ttl\n@@ -149,9 +153,6 @@\n                 if self.debug:\n                     self.r_server.incr('web2py_cache_statistics:hit_total')\n                 value = pickle.loads(obj)\n-            elif f is None:\n-                # delete and never look back\n-                self.r_server.delete(newKey)\n             else:\n                 # naive distributed locking\n                 if with_lock:\n", "issue": "RedisCache fails to delete cached objects when f=None\nFor background, see https://groups.google.com/d/msg/web2py/1LoXXrPlIGE/bX2u3CJ2BQAJ.\n\nAccording to the book, calling `cache.model(key, f=None)` should cause `key` to be deleted from the cache. However, in `gluon.contrib.redis_cache.RedisClient.__call__`, setting `f=None` only deletes the cached object if it has expired.\n\nThis causes a problem with `@cache.action` because it relies on the `f=None` deletion method to delete pages that were cached with an invalid HTTP status.\n\nWe should change the behavior of the Redis cache to match that of the other cache models (i.e., setting `f=None` should _always_ delete the key from the cache).\n\nAlso, for extra safety, perhaps `@cache.action` should delete pages with an invalid status via `cache_model.clear(r'^%s$' % cache_key)` rather than `cache_model(cache_key, None)`.\n\n", "before_files": [{"content": "\"\"\"\nDeveloped by niphlod@gmail.com\nReleased under web2py license because includes gluon/cache.py source code\n\"\"\"\n\ntry:\n    import cPickle as pickle\nexcept:\n    import pickle\nimport time\nimport re\nimport logging\nimport thread\nimport random\nfrom gluon import current\nfrom gluon.cache import CacheAbstract\nfrom gluon.contrib.redis_utils import acquire_lock, release_lock\nfrom gluon.contrib.redis_utils import register_release_lock, RConnectionError\n\nlogger = logging.getLogger(\"web2py.cache.redis\")\n\nlocker = thread.allocate_lock()\n\n\ndef RedisCache(redis_conn=None, debug=False, with_lock=False, fail_gracefully=False, db=None):\n    \"\"\"\n    Usage example: put in models::\n\n    First of all install Redis\n    Ubuntu :\n    sudo apt-get install redis-server\n    sudo pip install redis\n\n    Then\n\n        from gluon.contrib.redis_utils import RConn\n        rconn = RConn()\n        from gluon.contrib.redis_cache import RedisCache\n        cache.redis = RedisCache(redis_conn=rconn, debug=True, with_lock=True)\n\n    Args:\n        redis_conn: a redis-like connection object\n        debug: if True adds to stats() the total_hits and misses\n        with_lock: sets the default locking mode for creating new keys.\n            By default is False (usualy when you choose Redis you do it\n            for performances reason)\n            When True, only one thread/process can set a value concurrently\n        fail_gracefully: if redis is unavailable, returns the value computing it\n            instead of raising an exception\n\n    It can be used pretty much the same as cache.ram()\n    When you use cache.redis directly you can use :\n\n        redis_key_and_var_name = cache.redis('redis_key_and_var_name', lambda or function,\n                                             time_expire=time.time(), with_lock=True)\n\n    to enforce locking. The with_lock parameter overrides the one set in the\n    cache.redis instance creation\n\n    cache.redis.stats()\n        returns a dictionary with statistics of Redis server\n        with one additional key ('w2p_keys') showing all keys currently set\n        from web2py with their TTL\n\n    A little wording on how keys are stored (and why the cache_it() function\n    and the clear() one look a little bit convoluted): there are a lot of\n    libraries that just store values and then use the KEYS command to delete it.\n    Until recent releases of this module, that technique was used here too.\n    In the need of deleting specific keys in a database with zillions keys in it\n    (other web2py apps, other applications in the need of a Redis stack) the\n    KEYS command is slow (it needs to scan every key in the database).\n    So, we use Redis 'sets' to store keys in \"buckets\"...\n    - every key created gets \"indexed\" in a bucket\n    - all buckets are indexed in a fixed key that never expires\n    - all keys generated within the same minute go in the same bucket\n    - every bucket is then set to expire when every key within it is expired\n    When we need to clear() cached keys:\n    - we tell Redis to SUNION all buckets\n       - gives us just the keys that are not expired yet\n    - buckets that are expired are removed from the fixed set\n    - we scan the keys and then delete them\n    \"\"\"\n\n    locker.acquire()\n    try:\n        instance_name = 'redis_instance_' + current.request.application\n        if not hasattr(RedisCache, instance_name):\n            setattr(RedisCache, instance_name, \n                    RedisClient(redis_conn=redis_conn, debug=debug,\n                                with_lock=with_lock, fail_gracefully=fail_gracefully))\n        return getattr(RedisCache, instance_name)\n    finally:\n        locker.release()\n\n\nclass RedisClient(object):\n\n    meta_storage = {}\n    MAX_RETRIES = 5\n    RETRIES = 0\n\n    def __init__(self, redis_conn=None, debug=False,\n                 with_lock=False, fail_gracefully=False):\n        self.request = current.request\n        self.debug = debug\n        self.with_lock = with_lock\n        self.fail_gracefully = fail_gracefully\n        self.prefix = \"w2p:cache:%s:\" % self.request.application\n        if self.request:\n            app = self.request.application\n        else:\n            app = ''\n\n        if app not in self.meta_storage:\n            self.storage = self.meta_storage[app] = {\n                CacheAbstract.cache_stats_name: {\n                    'hit_total': 0,\n                    'misses': 0,\n                }}\n        else:\n            self.storage = self.meta_storage[app]\n\n        self.cache_set_key = 'w2p:%s:___cache_set' % self.request.application\n\n        self.r_server = redis_conn\n        self._release_script = register_release_lock(self.r_server)\n\n    def initialize(self):\n        pass\n\n    def __call__(self, key, f, time_expire=300, with_lock=None):\n        if with_lock is None:\n            with_lock = self.with_lock\n        if time_expire is None:\n            time_expire = 24 * 60 * 60\n        newKey = self.__keyFormat__(key)\n        value = None\n        ttl = 0\n        try:\n            # is there a value\n            obj = self.r_server.get(newKey)\n            # what's its ttl\n            if obj:\n                ttl = self.r_server.ttl(newKey)\n            if ttl > time_expire:\n                obj = None\n            if obj:\n                # was cached\n                if self.debug:\n                    self.r_server.incr('web2py_cache_statistics:hit_total')\n                value = pickle.loads(obj)\n            elif f is None:\n                # delete and never look back\n                self.r_server.delete(newKey)\n            else:\n                # naive distributed locking\n                if with_lock:\n                    lock_key = '%s:__lock' % newKey\n                    randomvalue = time.time()\n                    al = acquire_lock(self.r_server, lock_key, randomvalue)\n                    # someone may have computed it\n                    obj = self.r_server.get(newKey)\n                    if obj is None:\n                        value = self.cache_it(newKey, f, time_expire)\n                    else:\n                        value = pickle.loads(obj)\n                    release_lock(self, lock_key, al)\n                else:\n                    # without distributed locking\n                    value = self.cache_it(newKey, f, time_expire)\n            return value\n        except RConnectionError:\n            return self.retry_call(key, f, time_expire, with_lock)\n\n    def cache_it(self, key, f, time_expire):\n        if self.debug:\n            self.r_server.incr('web2py_cache_statistics:misses')\n        cache_set_key = self.cache_set_key\n        expire_at = int(time.time() + time_expire) + 120\n        bucket_key = \"%s:%s\" % (cache_set_key, expire_at / 60)\n        value = f()\n        value_ = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)\n        if time_expire == 0:\n            time_expire = 1\n        self.r_server.setex(key, time_expire, value_)\n        # print '%s will expire on %s: it goes in bucket %s' % (key, time.ctime(expire_at))\n        # print 'that will expire on %s' % (bucket_key, time.ctime(((expire_at / 60) + 1) * 60))\n        p = self.r_server.pipeline()\n        # add bucket to the fixed set\n        p.sadd(cache_set_key, bucket_key)\n        # sets the key\n        p.setex(key, time_expire, value_)\n        # add the key to the bucket\n        p.sadd(bucket_key, key)\n        # expire the bucket properly\n        p.expireat(bucket_key, ((expire_at / 60) + 1) * 60)\n        p.execute()\n        return value\n\n    def retry_call(self, key, f, time_expire, with_lock):\n        self.RETRIES += 1\n        if self.RETRIES <= self.MAX_RETRIES:\n            logger.error(\"sleeping %s seconds before reconnecting\" % (2 * self.RETRIES))\n            time.sleep(2 * self.RETRIES)\n            if self.fail_gracefully:\n                self.RETRIES = 0\n                return f()\n            return self.__call__(key, f, time_expire, with_lock)\n        else:\n            self.RETRIES = 0\n            if self.fail_gracefully:\n                return f\n            raise RConnectionError('Redis instance is unavailable')\n\n    def increment(self, key, value=1):\n        try:\n            newKey = self.__keyFormat__(key)\n            return self.r_server.incr(newKey, value)\n        except RConnectionError:\n            return self.retry_increment(key, value)\n\n    def retry_increment(self, key, value):\n        self.RETRIES += 1\n        if self.RETRIES <= self.MAX_RETRIES:\n            logger.error(\"sleeping some seconds before reconnecting\")\n            time.sleep(2 * self.RETRIES)\n            return self.increment(key, value)\n        else:\n            self.RETRIES = 0\n            raise RConnectionError('Redis instance is unavailable')\n\n    def clear(self, regex):\n        \"\"\"\n        Auxiliary function called by `clear` to search and\n        clear cache entries\n        \"\"\"\n        r = re.compile(regex)\n        # get all buckets\n        buckets = self.r_server.smembers(self.cache_set_key)\n        # get all keys in buckets\n        if buckets:\n            keys = self.r_server.sunion(buckets)\n        else:\n            return\n        prefix = self.prefix\n        pipe = self.r_server.pipeline()\n        for a in keys:\n            if r.match(str(a).replace(prefix, '', 1)):\n                pipe.delete(a)\n        if random.randrange(0, 100) < 10:\n            # do this just once in a while (10% chance)\n            self.clear_buckets(buckets)\n        pipe.execute()\n\n    def clear_buckets(self, buckets):\n        p = self.r_server.pipeline()\n        for b in buckets:\n            if not self.r_server.exists(b):\n                p.srem(self.cache_set_key, b)\n        p.execute()\n\n    def delete(self, key):\n        newKey = self.__keyFormat__(key)\n        return self.r_server.delete(newKey)\n\n    def stats(self):\n        stats_collector = self.r_server.info()\n        if self.debug:\n            stats_collector['w2p_stats'] = dict(\n                hit_total=self.r_server.get(\n                    'web2py_cache_statistics:hit_total'),\n                misses=self.r_server.get('web2py_cache_statistics:misses')\n            )\n        stats_collector['w2p_keys'] = dict()\n\n        for a in self.r_server.keys(\"w2p:%s:*\" % (\n                self.request.application)):\n            stats_collector['w2p_keys'][\"%s_expire_in_sec\" % a] = self.r_server.ttl(a)\n        return stats_collector\n\n    def __keyFormat__(self, key):\n        return '%s%s' % (self.prefix, key.replace(' ', '_'))\n", "path": "gluon/contrib/redis_cache.py"}], "after_files": [{"content": "\"\"\"\nDeveloped by niphlod@gmail.com\nReleased under web2py license because includes gluon/cache.py source code\n\"\"\"\n\ntry:\n    import cPickle as pickle\nexcept:\n    import pickle\nimport time\nimport re\nimport logging\nimport thread\nimport random\nfrom gluon import current\nfrom gluon.cache import CacheAbstract\nfrom gluon.contrib.redis_utils import acquire_lock, release_lock\nfrom gluon.contrib.redis_utils import register_release_lock, RConnectionError\n\nlogger = logging.getLogger(\"web2py.cache.redis\")\n\nlocker = thread.allocate_lock()\n\n\ndef RedisCache(redis_conn=None, debug=False, with_lock=False, fail_gracefully=False, db=None):\n    \"\"\"\n    Usage example: put in models::\n\n    First of all install Redis\n    Ubuntu :\n    sudo apt-get install redis-server\n    sudo pip install redis\n\n    Then\n\n        from gluon.contrib.redis_utils import RConn\n        rconn = RConn()\n        from gluon.contrib.redis_cache import RedisCache\n        cache.redis = RedisCache(redis_conn=rconn, debug=True, with_lock=True)\n\n    Args:\n        redis_conn: a redis-like connection object\n        debug: if True adds to stats() the total_hits and misses\n        with_lock: sets the default locking mode for creating new keys.\n            By default is False (usualy when you choose Redis you do it\n            for performances reason)\n            When True, only one thread/process can set a value concurrently\n        fail_gracefully: if redis is unavailable, returns the value computing it\n            instead of raising an exception\n\n    It can be used pretty much the same as cache.ram()\n    When you use cache.redis directly you can use :\n\n        redis_key_and_var_name = cache.redis('redis_key_and_var_name', lambda or function,\n                                             time_expire=time.time(), with_lock=True)\n\n    to enforce locking. The with_lock parameter overrides the one set in the\n    cache.redis instance creation\n\n    cache.redis.stats()\n        returns a dictionary with statistics of Redis server\n        with one additional key ('w2p_keys') showing all keys currently set\n        from web2py with their TTL\n\n    A little wording on how keys are stored (and why the cache_it() function\n    and the clear() one look a little bit convoluted): there are a lot of\n    libraries that just store values and then use the KEYS command to delete it.\n    Until recent releases of this module, that technique was used here too.\n    In the need of deleting specific keys in a database with zillions keys in it\n    (other web2py apps, other applications in the need of a Redis stack) the\n    KEYS command is slow (it needs to scan every key in the database).\n    So, we use Redis 'sets' to store keys in \"buckets\"...\n    - every key created gets \"indexed\" in a bucket\n    - all buckets are indexed in a fixed key that never expires\n    - all keys generated within the same minute go in the same bucket\n    - every bucket is then set to expire when every key within it is expired\n    When we need to clear() cached keys:\n    - we tell Redis to SUNION all buckets\n       - gives us just the keys that are not expired yet\n    - buckets that are expired are removed from the fixed set\n    - we scan the keys and then delete them\n    \"\"\"\n\n    locker.acquire()\n    try:\n        instance_name = 'redis_instance_' + current.request.application\n        if not hasattr(RedisCache, instance_name):\n            setattr(RedisCache, instance_name,\n                    RedisClient(redis_conn=redis_conn, debug=debug,\n                                with_lock=with_lock, fail_gracefully=fail_gracefully))\n        return getattr(RedisCache, instance_name)\n    finally:\n        locker.release()\n\n\nclass RedisClient(object):\n\n    meta_storage = {}\n    MAX_RETRIES = 5\n    RETRIES = 0\n\n    def __init__(self, redis_conn=None, debug=False,\n                 with_lock=False, fail_gracefully=False):\n        self.request = current.request\n        self.debug = debug\n        self.with_lock = with_lock\n        self.fail_gracefully = fail_gracefully\n        self.prefix = \"w2p:cache:%s:\" % self.request.application\n        if self.request:\n            app = self.request.application\n        else:\n            app = ''\n\n        if app not in self.meta_storage:\n            self.storage = self.meta_storage[app] = {\n                CacheAbstract.cache_stats_name: {\n                    'hit_total': 0,\n                    'misses': 0,\n                }}\n        else:\n            self.storage = self.meta_storage[app]\n\n        self.cache_set_key = 'w2p:%s:___cache_set' % self.request.application\n\n        self.r_server = redis_conn\n        self._release_script = register_release_lock(self.r_server)\n\n    def initialize(self):\n        pass\n\n    def __call__(self, key, f, time_expire=300, with_lock=None):\n        if with_lock is None:\n            with_lock = self.with_lock\n        if time_expire is None:\n            time_expire = 24 * 60 * 60\n        newKey = self.__keyFormat__(key)\n        value = None\n        ttl = 0\n        try:\n            if f is None:\n                # delete and never look back\n                self.r_server.delete(newKey)\n                return None\n            # is there a value\n            obj = self.r_server.get(newKey)\n            # what's its ttl\n            if obj:\n                ttl = self.r_server.ttl(newKey)\n            if ttl > time_expire:\n                obj = None\n            if obj:\n                # was cached\n                if self.debug:\n                    self.r_server.incr('web2py_cache_statistics:hit_total')\n                value = pickle.loads(obj)\n            else:\n                # naive distributed locking\n                if with_lock:\n                    lock_key = '%s:__lock' % newKey\n                    randomvalue = time.time()\n                    al = acquire_lock(self.r_server, lock_key, randomvalue)\n                    # someone may have computed it\n                    obj = self.r_server.get(newKey)\n                    if obj is None:\n                        value = self.cache_it(newKey, f, time_expire)\n                    else:\n                        value = pickle.loads(obj)\n                    release_lock(self, lock_key, al)\n                else:\n                    # without distributed locking\n                    value = self.cache_it(newKey, f, time_expire)\n            return value\n        except RConnectionError:\n            return self.retry_call(key, f, time_expire, with_lock)\n\n    def cache_it(self, key, f, time_expire):\n        if self.debug:\n            self.r_server.incr('web2py_cache_statistics:misses')\n        cache_set_key = self.cache_set_key\n        expire_at = int(time.time() + time_expire) + 120\n        bucket_key = \"%s:%s\" % (cache_set_key, expire_at / 60)\n        value = f()\n        value_ = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)\n        if time_expire == 0:\n            time_expire = 1\n        self.r_server.setex(key, time_expire, value_)\n        # print '%s will expire on %s: it goes in bucket %s' % (key, time.ctime(expire_at))\n        # print 'that will expire on %s' % (bucket_key, time.ctime(((expire_at / 60) + 1) * 60))\n        p = self.r_server.pipeline()\n        # add bucket to the fixed set\n        p.sadd(cache_set_key, bucket_key)\n        # sets the key\n        p.setex(key, time_expire, value_)\n        # add the key to the bucket\n        p.sadd(bucket_key, key)\n        # expire the bucket properly\n        p.expireat(bucket_key, ((expire_at / 60) + 1) * 60)\n        p.execute()\n        return value\n\n    def retry_call(self, key, f, time_expire, with_lock):\n        self.RETRIES += 1\n        if self.RETRIES <= self.MAX_RETRIES:\n            logger.error(\"sleeping %s seconds before reconnecting\" % (2 * self.RETRIES))\n            time.sleep(2 * self.RETRIES)\n            if self.fail_gracefully:\n                self.RETRIES = 0\n                return f()\n            return self.__call__(key, f, time_expire, with_lock)\n        else:\n            self.RETRIES = 0\n            if self.fail_gracefully:\n                return f\n            raise RConnectionError('Redis instance is unavailable')\n\n    def increment(self, key, value=1):\n        try:\n            newKey = self.__keyFormat__(key)\n            return self.r_server.incr(newKey, value)\n        except RConnectionError:\n            return self.retry_increment(key, value)\n\n    def retry_increment(self, key, value):\n        self.RETRIES += 1\n        if self.RETRIES <= self.MAX_RETRIES:\n            logger.error(\"sleeping some seconds before reconnecting\")\n            time.sleep(2 * self.RETRIES)\n            return self.increment(key, value)\n        else:\n            self.RETRIES = 0\n            raise RConnectionError('Redis instance is unavailable')\n\n    def clear(self, regex):\n        \"\"\"\n        Auxiliary function called by `clear` to search and\n        clear cache entries\n        \"\"\"\n        r = re.compile(regex)\n        # get all buckets\n        buckets = self.r_server.smembers(self.cache_set_key)\n        # get all keys in buckets\n        if buckets:\n            keys = self.r_server.sunion(buckets)\n        else:\n            return\n        prefix = self.prefix\n        pipe = self.r_server.pipeline()\n        for a in keys:\n            if r.match(str(a).replace(prefix, '', 1)):\n                pipe.delete(a)\n        if random.randrange(0, 100) < 10:\n            # do this just once in a while (10% chance)\n            self.clear_buckets(buckets)\n        pipe.execute()\n\n    def clear_buckets(self, buckets):\n        p = self.r_server.pipeline()\n        for b in buckets:\n            if not self.r_server.exists(b):\n                p.srem(self.cache_set_key, b)\n        p.execute()\n\n    def delete(self, key):\n        newKey = self.__keyFormat__(key)\n        return self.r_server.delete(newKey)\n\n    def stats(self):\n        stats_collector = self.r_server.info()\n        if self.debug:\n            stats_collector['w2p_stats'] = dict(\n                hit_total=self.r_server.get(\n                    'web2py_cache_statistics:hit_total'),\n                misses=self.r_server.get('web2py_cache_statistics:misses')\n            )\n        stats_collector['w2p_keys'] = dict()\n\n        for a in self.r_server.keys(\"w2p:%s:*\" % (\n                self.request.application)):\n            stats_collector['w2p_keys'][\"%s_expire_in_sec\" % a] = self.r_server.ttl(a)\n        return stats_collector\n\n    def __keyFormat__(self, key):\n        return '%s%s' % (self.prefix, key.replace(' ', '_'))\n", "path": "gluon/contrib/redis_cache.py"}]}