We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
KeyError in robotstxt middleware
I'm getting these errors in robots.txt middleware:

```
2016-01-27 16:18:21 [scrapy.core.scraper] ERROR: Error downloading <GET http://yellowpages.co.th>
Traceback (most recent call last):
  File "/Users/kmike/envs/scraping/lib/python2.7/site-packages/twisted/internet/defer.py", line 150, in maybeDeferred
    result = f(*args, **kw)
  File "/Users/kmike/svn/scrapy/scrapy/downloadermiddlewares/robotstxt.py", line 65, in robot_parser
    if isinstance(self._parsers[netloc], Deferred):
KeyError: 'yellowpages.co.th'
```

It looks like https://github.com/scrapy/scrapy/pull/1473 caused it (I can't get this issue in Scrapy 1.0.4, but it present in Scrapy master). It happens when page failed to download and HTTP cache is enabled. I haven't debugged it further.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/downloadermiddlewares/robotstxt.py`
Content:
```
1 """
2 This is a middleware to respect robots.txt policies. To activate it you must
3 enable this middleware and enable the ROBOTSTXT_OBEY setting.
4 
5 """
6 
7 import logging
8 
9 from six.moves.urllib import robotparser
10 
11 from twisted.internet.defer import Deferred, maybeDeferred
12 from scrapy.exceptions import NotConfigured, IgnoreRequest
13 from scrapy.http import Request
14 from scrapy.utils.httpobj import urlparse_cached
15 from scrapy.utils.log import failure_to_exc_info
16 
17 logger = logging.getLogger(__name__)
18 
19 
20 class RobotsTxtMiddleware(object):
21     DOWNLOAD_PRIORITY = 1000
22 
23     def __init__(self, crawler):
24         if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
25             raise NotConfigured
26 
27         self.crawler = crawler
28         self._useragent = crawler.settings.get('USER_AGENT')
29         self._parsers = {}
30 
31     @classmethod
32     def from_crawler(cls, crawler):
33         return cls(crawler)
34 
35     def process_request(self, request, spider):
36         if request.meta.get('dont_obey_robotstxt'):
37             return
38         d = maybeDeferred(self.robot_parser, request, spider)
39         d.addCallback(self.process_request_2, request, spider)
40         return d
41 
42     def process_request_2(self, rp, request, spider):
43         if rp is not None and not rp.can_fetch(self._useragent, request.url):
44             logger.debug("Forbidden by robots.txt: %(request)s",
45                          {'request': request}, extra={'spider': spider})
46             raise IgnoreRequest()
47 
48     def robot_parser(self, request, spider):
49         url = urlparse_cached(request)
50         netloc = url.netloc
51 
52         if netloc not in self._parsers:
53             self._parsers[netloc] = Deferred()
54             robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
55             robotsreq = Request(
56                 robotsurl,
57                 priority=self.DOWNLOAD_PRIORITY,
58                 meta={'dont_obey_robotstxt': True}
59             )
60             dfd = self.crawler.engine.download(robotsreq, spider)
61             dfd.addCallback(self._parse_robots, netloc)
62             dfd.addErrback(self._logerror, robotsreq, spider)
63             dfd.addErrback(self._robots_error, netloc)
64 
65         if isinstance(self._parsers[netloc], Deferred):
66             d = Deferred()
67             def cb(result):
68                 d.callback(result)
69                 return result
70             self._parsers[netloc].addCallback(cb)
71             return d
72         else:
73             return self._parsers[netloc]
74 
75     def _logerror(self, failure, request, spider):
76         if failure.type is not IgnoreRequest:
77             logger.error("Error downloading %(request)s: %(f_exception)s",
78                          {'request': request, 'f_exception': failure.value},
79                          exc_info=failure_to_exc_info(failure),
80                          extra={'spider': spider})
81         return failure
82 
83     def _parse_robots(self, response, netloc):
84         rp = robotparser.RobotFileParser(response.url)
85         body = ''
86         if hasattr(response, 'text'):
87             body = response.text
88         else: # last effort try
89             try:
90                 body = response.body.decode('utf-8')
91             except UnicodeDecodeError:
92                 # If we found garbage, disregard it:,
93                 # but keep the lookup cached (in self._parsers)
94                 # Running rp.parse() will set rp state from
95                 # 'disallow all' to 'allow any'.
96                 pass
97         rp.parse(body.splitlines())
98 
99         rp_dfd = self._parsers[netloc]
100         self._parsers[netloc] = rp
101         rp_dfd.callback(rp)
102 
103     def _robots_error(self, failure, netloc):
104         self._parsers.pop(netloc).callback(None)
105
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -101,4 +101,6 @@
         rp_dfd.callback(rp)
 
     def _robots_error(self, failure, netloc):
-        self._parsers.pop(netloc).callback(None)
+        rp_dfd = self._parsers[netloc]
+        self._parsers[netloc] = None
+        rp_dfd.callback(None)

{"golden_diff": "diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py\n--- a/scrapy/downloadermiddlewares/robotstxt.py\n+++ b/scrapy/downloadermiddlewares/robotstxt.py\n@@ -101,4 +101,6 @@\n         rp_dfd.callback(rp)\n \n     def _robots_error(self, failure, netloc):\n-        self._parsers.pop(netloc).callback(None)\n+        rp_dfd = self._parsers[netloc]\n+        self._parsers[netloc] = None\n+        rp_dfd.callback(None)\n", "issue": "KeyError in robotstxt middleware\nI'm getting these errors in robots.txt middleware:\n\n```\n2016-01-27 16:18:21 [scrapy.core.scraper] ERROR: Error downloading <GET http://yellowpages.co.th>\nTraceback (most recent call last):\n  File \"/Users/kmike/envs/scraping/lib/python2.7/site-packages/twisted/internet/defer.py\", line 150, in maybeDeferred\n    result = f(*args, **kw)\n  File \"/Users/kmike/svn/scrapy/scrapy/downloadermiddlewares/robotstxt.py\", line 65, in robot_parser\n    if isinstance(self._parsers[netloc], Deferred):\nKeyError: 'yellowpages.co.th'\n```\n\nIt looks like https://github.com/scrapy/scrapy/pull/1473 caused it (I can't get this issue in Scrapy 1.0.4, but it present in Scrapy master). It happens when page failed to download and HTTP cache is enabled. I haven't debugged it further.\n\n", "before_files": [{"content": "\"\"\"\nThis is a middleware to respect robots.txt policies. To activate it you must\nenable this middleware and enable the ROBOTSTXT_OBEY setting.\n\n\"\"\"\n\nimport logging\n\nfrom six.moves.urllib import robotparser\n\nfrom twisted.internet.defer import Deferred, maybeDeferred\nfrom scrapy.exceptions import NotConfigured, IgnoreRequest\nfrom scrapy.http import Request\nfrom scrapy.utils.httpobj import urlparse_cached\nfrom scrapy.utils.log import failure_to_exc_info\n\nlogger = logging.getLogger(__name__)\n\n\nclass RobotsTxtMiddleware(object):\n    DOWNLOAD_PRIORITY = 1000\n\n    def __init__(self, crawler):\n        if not crawler.settings.getbool('ROBOTSTXT_OBEY'):\n            raise NotConfigured\n\n        self.crawler = crawler\n        self._useragent = crawler.settings.get('USER_AGENT')\n        self._parsers = {}\n\n    @classmethod\n    def from_crawler(cls, crawler):\n        return cls(crawler)\n\n    def process_request(self, request, spider):\n        if request.meta.get('dont_obey_robotstxt'):\n            return\n        d = maybeDeferred(self.robot_parser, request, spider)\n        d.addCallback(self.process_request_2, request, spider)\n        return d\n\n    def process_request_2(self, rp, request, spider):\n        if rp is not None and not rp.can_fetch(self._useragent, request.url):\n            logger.debug(\"Forbidden by robots.txt: %(request)s\",\n                         {'request': request}, extra={'spider': spider})\n            raise IgnoreRequest()\n\n    def robot_parser(self, request, spider):\n        url = urlparse_cached(request)\n        netloc = url.netloc\n\n        if netloc not in self._parsers:\n            self._parsers[netloc] = Deferred()\n            robotsurl = \"%s://%s/robots.txt\" % (url.scheme, url.netloc)\n            robotsreq = Request(\n                robotsurl,\n                priority=self.DOWNLOAD_PRIORITY,\n                meta={'dont_obey_robotstxt': True}\n            )\n            dfd = self.crawler.engine.download(robotsreq, spider)\n            dfd.addCallback(self._parse_robots, netloc)\n            dfd.addErrback(self._logerror, robotsreq, spider)\n            dfd.addErrback(self._robots_error, netloc)\n\n        if isinstance(self._parsers[netloc], Deferred):\n            d = Deferred()\n            def cb(result):\n                d.callback(result)\n                return result\n            self._parsers[netloc].addCallback(cb)\n            return d\n        else:\n            return self._parsers[netloc]\n\n    def _logerror(self, failure, request, spider):\n        if failure.type is not IgnoreRequest:\n            logger.error(\"Error downloading %(request)s: %(f_exception)s\",\n                         {'request': request, 'f_exception': failure.value},\n                         exc_info=failure_to_exc_info(failure),\n                         extra={'spider': spider})\n        return failure\n\n    def _parse_robots(self, response, netloc):\n        rp = robotparser.RobotFileParser(response.url)\n        body = ''\n        if hasattr(response, 'text'):\n            body = response.text\n        else: # last effort try\n            try:\n                body = response.body.decode('utf-8')\n            except UnicodeDecodeError:\n                # If we found garbage, disregard it:,\n                # but keep the lookup cached (in self._parsers)\n                # Running rp.parse() will set rp state from\n                # 'disallow all' to 'allow any'.\n                pass\n        rp.parse(body.splitlines())\n\n        rp_dfd = self._parsers[netloc]\n        self._parsers[netloc] = rp\n        rp_dfd.callback(rp)\n\n    def _robots_error(self, failure, netloc):\n        self._parsers.pop(netloc).callback(None)\n", "path": "scrapy/downloadermiddlewares/robotstxt.py"}], "after_files": [{"content": "\"\"\"\nThis is a middleware to respect robots.txt policies. To activate it you must\nenable this middleware and enable the ROBOTSTXT_OBEY setting.\n\n\"\"\"\n\nimport logging\n\nfrom six.moves.urllib import robotparser\n\nfrom twisted.internet.defer import Deferred, maybeDeferred\nfrom scrapy.exceptions import NotConfigured, IgnoreRequest\nfrom scrapy.http import Request\nfrom scrapy.utils.httpobj import urlparse_cached\nfrom scrapy.utils.log import failure_to_exc_info\n\nlogger = logging.getLogger(__name__)\n\n\nclass RobotsTxtMiddleware(object):\n    DOWNLOAD_PRIORITY = 1000\n\n    def __init__(self, crawler):\n        if not crawler.settings.getbool('ROBOTSTXT_OBEY'):\n            raise NotConfigured\n\n        self.crawler = crawler\n        self._useragent = crawler.settings.get('USER_AGENT')\n        self._parsers = {}\n\n    @classmethod\n    def from_crawler(cls, crawler):\n        return cls(crawler)\n\n    def process_request(self, request, spider):\n        if request.meta.get('dont_obey_robotstxt'):\n            return\n        d = maybeDeferred(self.robot_parser, request, spider)\n        d.addCallback(self.process_request_2, request, spider)\n        return d\n\n    def process_request_2(self, rp, request, spider):\n        if rp is not None and not rp.can_fetch(self._useragent, request.url):\n            logger.debug(\"Forbidden by robots.txt: %(request)s\",\n                         {'request': request}, extra={'spider': spider})\n            raise IgnoreRequest()\n\n    def robot_parser(self, request, spider):\n        url = urlparse_cached(request)\n        netloc = url.netloc\n\n        if netloc not in self._parsers:\n            self._parsers[netloc] = Deferred()\n            robotsurl = \"%s://%s/robots.txt\" % (url.scheme, url.netloc)\n            robotsreq = Request(\n                robotsurl,\n                priority=self.DOWNLOAD_PRIORITY,\n                meta={'dont_obey_robotstxt': True}\n            )\n            dfd = self.crawler.engine.download(robotsreq, spider)\n            dfd.addCallback(self._parse_robots, netloc)\n            dfd.addErrback(self._logerror, robotsreq, spider)\n            dfd.addErrback(self._robots_error, netloc)\n\n        if isinstance(self._parsers[netloc], Deferred):\n            d = Deferred()\n            def cb(result):\n                d.callback(result)\n                return result\n            self._parsers[netloc].addCallback(cb)\n            return d\n        else:\n            return self._parsers[netloc]\n\n    def _logerror(self, failure, request, spider):\n        if failure.type is not IgnoreRequest:\n            logger.error(\"Error downloading %(request)s: %(f_exception)s\",\n                         {'request': request, 'f_exception': failure.value},\n                         exc_info=failure_to_exc_info(failure),\n                         extra={'spider': spider})\n        return failure\n\n    def _parse_robots(self, response, netloc):\n        rp = robotparser.RobotFileParser(response.url)\n        body = ''\n        if hasattr(response, 'body_as_unicode'):\n            body = response.body_as_unicode()\n        else: # last effort try\n            try:\n                body = response.body.decode('utf-8')\n            except UnicodeDecodeError:\n                # If we found garbage, disregard it:,\n                # but keep the lookup cached (in self._parsers)\n                # Running rp.parse() will set rp state from\n                # 'disallow all' to 'allow any'.\n                pass\n        rp.parse(body.splitlines())\n\n        rp_dfd = self._parsers[netloc]\n        self._parsers[netloc] = rp\n        rp_dfd.callback(rp)\n\n    def _robots_error(self, failure, netloc):\n        rp_dfd = self._parsers[netloc]\n        self._parsers[netloc] = None\n        rp_dfd.callback(None)\n", "path": "scrapy/downloadermiddlewares/robotstxt.py"}]}