We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
MediaPipeline exceptions passed silently
### Description

MediaPipeline exceptions passed silently both for method body or method signature errors.

### Steps to Reproduce

```
from scrapy.pipelines.files import FilesPipeline
class BuggyFilesPipeline(FilesPipeline):
    def file_path(self, request, response=None, info=None, *, item=None):
        return 1 / 0
```
**Expected behavior:** Exception logged

**Actual behavior:** Exception passed silently

**Reproduces how often:** 100%

### Versions

Scrapy 2.4


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/pipelines/media.py`
Content:
```
1 import functools
2 import logging
3 from collections import defaultdict
4 
5 from twisted.internet.defer import Deferred, DeferredList
6 from twisted.python.failure import Failure
7 
8 from scrapy.http.request import NO_CALLBACK
9 from scrapy.settings import Settings
10 from scrapy.utils.datatypes import SequenceExclude
11 from scrapy.utils.defer import defer_result, mustbe_deferred
12 from scrapy.utils.log import failure_to_exc_info
13 from scrapy.utils.misc import arg_to_iter
14 
15 logger = logging.getLogger(__name__)
16 
17 
18 def _DUMMY_CALLBACK(response):
19     return response
20 
21 
22 class MediaPipeline:
23     LOG_FAILED_RESULTS = True
24 
25     class SpiderInfo:
26         def __init__(self, spider):
27             self.spider = spider
28             self.downloading = set()
29             self.downloaded = {}
30             self.waiting = defaultdict(list)
31 
32     def __init__(self, download_func=None, settings=None):
33         self.download_func = download_func
34         self._expects_item = {}
35 
36         if isinstance(settings, dict) or settings is None:
37             settings = Settings(settings)
38         resolve = functools.partial(
39             self._key_for_pipe, base_class_name="MediaPipeline", settings=settings
40         )
41         self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False)
42         self._handle_statuses(self.allow_redirects)
43 
44     def _handle_statuses(self, allow_redirects):
45         self.handle_httpstatus_list = None
46         if allow_redirects:
47             self.handle_httpstatus_list = SequenceExclude(range(300, 400))
48 
49     def _key_for_pipe(self, key, base_class_name=None, settings=None):
50         """
51         >>> MediaPipeline()._key_for_pipe("IMAGES")
52         'IMAGES'
53         >>> class MyPipe(MediaPipeline):
54         ...     pass
55         >>> MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
56         'MYPIPE_IMAGES'
57         """
58         class_name = self.__class__.__name__
59         formatted_key = f"{class_name.upper()}_{key}"
60         if (
61             not base_class_name
62             or class_name == base_class_name
63             or settings
64             and not settings.get(formatted_key)
65         ):
66             return key
67         return formatted_key
68 
69     @classmethod
70     def from_crawler(cls, crawler):
71         try:
72             pipe = cls.from_settings(crawler.settings)
73         except AttributeError:
74             pipe = cls()
75         pipe.crawler = crawler
76         pipe._fingerprinter = crawler.request_fingerprinter
77         return pipe
78 
79     def open_spider(self, spider):
80         self.spiderinfo = self.SpiderInfo(spider)
81 
82     def process_item(self, item, spider):
83         info = self.spiderinfo
84         requests = arg_to_iter(self.get_media_requests(item, info))
85         dlist = [self._process_request(r, info, item) for r in requests]
86         dfd = DeferredList(dlist, consumeErrors=True)
87         return dfd.addCallback(self.item_completed, item, info)
88 
89     def _process_request(self, request, info, item):
90         fp = self._fingerprinter.fingerprint(request)
91         if not request.callback or request.callback is NO_CALLBACK:
92             cb = _DUMMY_CALLBACK
93         else:
94             cb = request.callback
95         eb = request.errback
96         request.callback = NO_CALLBACK
97         request.errback = None
98 
99         # Return cached result if request was already seen
100         if fp in info.downloaded:
101             return defer_result(info.downloaded[fp]).addCallbacks(cb, eb)
102 
103         # Otherwise, wait for result
104         wad = Deferred().addCallbacks(cb, eb)
105         info.waiting[fp].append(wad)
106 
107         # Check if request is downloading right now to avoid doing it twice
108         if fp in info.downloading:
109             return wad
110 
111         # Download request checking media_to_download hook output first
112         info.downloading.add(fp)
113         dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
114         dfd.addCallback(self._check_media_to_download, request, info, item=item)
115         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
116         dfd.addErrback(
117             lambda f: logger.error(
118                 f.value, exc_info=failure_to_exc_info(f), extra={"spider": info.spider}
119             )
120         )
121         return dfd.addBoth(lambda _: wad)  # it must return wad at last
122 
123     def _modify_media_request(self, request):
124         if self.handle_httpstatus_list:
125             request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list
126         else:
127             request.meta["handle_httpstatus_all"] = True
128 
129     def _check_media_to_download(self, result, request, info, item):
130         if result is not None:
131             return result
132         if self.download_func:
133             # this ugly code was left only to support tests. TODO: remove
134             dfd = mustbe_deferred(self.download_func, request, info.spider)
135             dfd.addCallbacks(
136                 callback=self.media_downloaded,
137                 callbackArgs=(request, info),
138                 callbackKeywords={"item": item},
139                 errback=self.media_failed,
140                 errbackArgs=(request, info),
141             )
142         else:
143             self._modify_media_request(request)
144             dfd = self.crawler.engine.download(request)
145             dfd.addCallbacks(
146                 callback=self.media_downloaded,
147                 callbackArgs=(request, info),
148                 callbackKeywords={"item": item},
149                 errback=self.media_failed,
150                 errbackArgs=(request, info),
151             )
152         return dfd
153 
154     def _cache_result_and_execute_waiters(self, result, fp, info):
155         if isinstance(result, Failure):
156             # minimize cached information for failure
157             result.cleanFailure()
158             result.frames = []
159             result.stack = None
160 
161             # This code fixes a memory leak by avoiding to keep references to
162             # the Request and Response objects on the Media Pipeline cache.
163             #
164             # What happens when the media_downloaded callback raises an
165             # exception, for example a FileException('download-error') when
166             # the Response status code is not 200 OK, is that the original
167             # StopIteration exception (which in turn contains the failed
168             # Response and by extension, the original Request) gets encapsulated
169             # within the FileException context.
170             #
171             # Originally, Scrapy was using twisted.internet.defer.returnValue
172             # inside functions decorated with twisted.internet.defer.inlineCallbacks,
173             # encapsulating the returned Response in a _DefGen_Return exception
174             # instead of a StopIteration.
175             #
176             # To avoid keeping references to the Response and therefore Request
177             # objects on the Media Pipeline cache, we should wipe the context of
178             # the encapsulated exception when it is a StopIteration instance
179             #
180             # This problem does not occur in Python 2.7 since we don't have
181             # Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
182             context = getattr(result.value, "__context__", None)
183             if isinstance(context, StopIteration):
184                 setattr(result.value, "__context__", None)
185 
186         info.downloading.remove(fp)
187         info.downloaded[fp] = result  # cache result
188         for wad in info.waiting.pop(fp):
189             defer_result(result).chainDeferred(wad)
190 
191     # Overridable Interface
192     def media_to_download(self, request, info, *, item=None):
193         """Check request before starting download"""
194         pass
195 
196     def get_media_requests(self, item, info):
197         """Returns the media requests to download"""
198         pass
199 
200     def media_downloaded(self, response, request, info, *, item=None):
201         """Handler for success downloads"""
202         return response
203 
204     def media_failed(self, failure, request, info):
205         """Handler for failed downloads"""
206         return failure
207 
208     def item_completed(self, results, item, info):
209         """Called per item when all media requests has been processed"""
210         if self.LOG_FAILED_RESULTS:
211             for ok, value in results:
212                 if not ok:
213                     logger.error(
214                         "%(class)s found errors processing %(item)s",
215                         {"class": self.__class__.__name__, "item": item},
216                         exc_info=failure_to_exc_info(value),
217                         extra={"spider": info.spider},
218                     )
219         return item
220 
221     def file_path(self, request, response=None, info=None, *, item=None):
222         """Returns the path where downloaded media should be stored"""
223         pass
224
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -112,14 +112,14 @@
         info.downloading.add(fp)
         dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
         dfd.addCallback(self._check_media_to_download, request, info, item=item)
+        dfd.addErrback(self._log_exception)
         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
-        dfd.addErrback(
-            lambda f: logger.error(
-                f.value, exc_info=failure_to_exc_info(f), extra={"spider": info.spider}
-            )
-        )
         return dfd.addBoth(lambda _: wad)  # it must return wad at last
 
+    def _log_exception(self, result):
+        logger.exception(result)
+        return result
+
     def _modify_media_request(self, request):
         if self.handle_httpstatus_list:
             request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list

{"golden_diff": "diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py\n--- a/scrapy/pipelines/media.py\n+++ b/scrapy/pipelines/media.py\n@@ -112,14 +112,14 @@\n         info.downloading.add(fp)\n         dfd = mustbe_deferred(self.media_to_download, request, info, item=item)\n         dfd.addCallback(self._check_media_to_download, request, info, item=item)\n+        dfd.addErrback(self._log_exception)\n         dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)\n-        dfd.addErrback(\n-            lambda f: logger.error(\n-                f.value, exc_info=failure_to_exc_info(f), extra={\"spider\": info.spider}\n-            )\n-        )\n         return dfd.addBoth(lambda _: wad)  # it must return wad at last\n \n+    def _log_exception(self, result):\n+        logger.exception(result)\n+        return result\n+\n     def _modify_media_request(self, request):\n         if self.handle_httpstatus_list:\n             request.meta[\"handle_httpstatus_list\"] = self.handle_httpstatus_list\n", "issue": "MediaPipeline exceptions passed silently\n### Description\r\n\r\nMediaPipeline exceptions passed silently both for method body or method signature errors.\r\n\r\n### Steps to Reproduce\r\n\r\n```\r\nfrom scrapy.pipelines.files import FilesPipeline\r\nclass BuggyFilesPipeline(FilesPipeline):\r\n    def file_path(self, request, response=None, info=None, *, item=None):\r\n        return 1 / 0\r\n```\r\n**Expected behavior:** Exception logged\r\n\r\n**Actual behavior:** Exception passed silently\r\n\r\n**Reproduces how often:** 100%\r\n\r\n### Versions\r\n\r\nScrapy 2.4\r\n\n", "before_files": [{"content": "import functools\nimport logging\nfrom collections import defaultdict\n\nfrom twisted.internet.defer import Deferred, DeferredList\nfrom twisted.python.failure import Failure\n\nfrom scrapy.http.request import NO_CALLBACK\nfrom scrapy.settings import Settings\nfrom scrapy.utils.datatypes import SequenceExclude\nfrom scrapy.utils.defer import defer_result, mustbe_deferred\nfrom scrapy.utils.log import failure_to_exc_info\nfrom scrapy.utils.misc import arg_to_iter\n\nlogger = logging.getLogger(__name__)\n\n\ndef _DUMMY_CALLBACK(response):\n    return response\n\n\nclass MediaPipeline:\n    LOG_FAILED_RESULTS = True\n\n    class SpiderInfo:\n        def __init__(self, spider):\n            self.spider = spider\n            self.downloading = set()\n            self.downloaded = {}\n            self.waiting = defaultdict(list)\n\n    def __init__(self, download_func=None, settings=None):\n        self.download_func = download_func\n        self._expects_item = {}\n\n        if isinstance(settings, dict) or settings is None:\n            settings = Settings(settings)\n        resolve = functools.partial(\n            self._key_for_pipe, base_class_name=\"MediaPipeline\", settings=settings\n        )\n        self.allow_redirects = settings.getbool(resolve(\"MEDIA_ALLOW_REDIRECTS\"), False)\n        self._handle_statuses(self.allow_redirects)\n\n    def _handle_statuses(self, allow_redirects):\n        self.handle_httpstatus_list = None\n        if allow_redirects:\n            self.handle_httpstatus_list = SequenceExclude(range(300, 400))\n\n    def _key_for_pipe(self, key, base_class_name=None, settings=None):\n        \"\"\"\n        >>> MediaPipeline()._key_for_pipe(\"IMAGES\")\n        'IMAGES'\n        >>> class MyPipe(MediaPipeline):\n        ...     pass\n        >>> MyPipe()._key_for_pipe(\"IMAGES\", base_class_name=\"MediaPipeline\")\n        'MYPIPE_IMAGES'\n        \"\"\"\n        class_name = self.__class__.__name__\n        formatted_key = f\"{class_name.upper()}_{key}\"\n        if (\n            not base_class_name\n            or class_name == base_class_name\n            or settings\n            and not settings.get(formatted_key)\n        ):\n            return key\n        return formatted_key\n\n    @classmethod\n    def from_crawler(cls, crawler):\n        try:\n            pipe = cls.from_settings(crawler.settings)\n        except AttributeError:\n            pipe = cls()\n        pipe.crawler = crawler\n        pipe._fingerprinter = crawler.request_fingerprinter\n        return pipe\n\n    def open_spider(self, spider):\n        self.spiderinfo = self.SpiderInfo(spider)\n\n    def process_item(self, item, spider):\n        info = self.spiderinfo\n        requests = arg_to_iter(self.get_media_requests(item, info))\n        dlist = [self._process_request(r, info, item) for r in requests]\n        dfd = DeferredList(dlist, consumeErrors=True)\n        return dfd.addCallback(self.item_completed, item, info)\n\n    def _process_request(self, request, info, item):\n        fp = self._fingerprinter.fingerprint(request)\n        if not request.callback or request.callback is NO_CALLBACK:\n            cb = _DUMMY_CALLBACK\n        else:\n            cb = request.callback\n        eb = request.errback\n        request.callback = NO_CALLBACK\n        request.errback = None\n\n        # Return cached result if request was already seen\n        if fp in info.downloaded:\n            return defer_result(info.downloaded[fp]).addCallbacks(cb, eb)\n\n        # Otherwise, wait for result\n        wad = Deferred().addCallbacks(cb, eb)\n        info.waiting[fp].append(wad)\n\n        # Check if request is downloading right now to avoid doing it twice\n        if fp in info.downloading:\n            return wad\n\n        # Download request checking media_to_download hook output first\n        info.downloading.add(fp)\n        dfd = mustbe_deferred(self.media_to_download, request, info, item=item)\n        dfd.addCallback(self._check_media_to_download, request, info, item=item)\n        dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)\n        dfd.addErrback(\n            lambda f: logger.error(\n                f.value, exc_info=failure_to_exc_info(f), extra={\"spider\": info.spider}\n            )\n        )\n        return dfd.addBoth(lambda _: wad)  # it must return wad at last\n\n    def _modify_media_request(self, request):\n        if self.handle_httpstatus_list:\n            request.meta[\"handle_httpstatus_list\"] = self.handle_httpstatus_list\n        else:\n            request.meta[\"handle_httpstatus_all\"] = True\n\n    def _check_media_to_download(self, result, request, info, item):\n        if result is not None:\n            return result\n        if self.download_func:\n            # this ugly code was left only to support tests. TODO: remove\n            dfd = mustbe_deferred(self.download_func, request, info.spider)\n            dfd.addCallbacks(\n                callback=self.media_downloaded,\n                callbackArgs=(request, info),\n                callbackKeywords={\"item\": item},\n                errback=self.media_failed,\n                errbackArgs=(request, info),\n            )\n        else:\n            self._modify_media_request(request)\n            dfd = self.crawler.engine.download(request)\n            dfd.addCallbacks(\n                callback=self.media_downloaded,\n                callbackArgs=(request, info),\n                callbackKeywords={\"item\": item},\n                errback=self.media_failed,\n                errbackArgs=(request, info),\n            )\n        return dfd\n\n    def _cache_result_and_execute_waiters(self, result, fp, info):\n        if isinstance(result, Failure):\n            # minimize cached information for failure\n            result.cleanFailure()\n            result.frames = []\n            result.stack = None\n\n            # This code fixes a memory leak by avoiding to keep references to\n            # the Request and Response objects on the Media Pipeline cache.\n            #\n            # What happens when the media_downloaded callback raises an\n            # exception, for example a FileException('download-error') when\n            # the Response status code is not 200 OK, is that the original\n            # StopIteration exception (which in turn contains the failed\n            # Response and by extension, the original Request) gets encapsulated\n            # within the FileException context.\n            #\n            # Originally, Scrapy was using twisted.internet.defer.returnValue\n            # inside functions decorated with twisted.internet.defer.inlineCallbacks,\n            # encapsulating the returned Response in a _DefGen_Return exception\n            # instead of a StopIteration.\n            #\n            # To avoid keeping references to the Response and therefore Request\n            # objects on the Media Pipeline cache, we should wipe the context of\n            # the encapsulated exception when it is a StopIteration instance\n            #\n            # This problem does not occur in Python 2.7 since we don't have\n            # Exception Chaining (https://www.python.org/dev/peps/pep-3134/).\n            context = getattr(result.value, \"__context__\", None)\n            if isinstance(context, StopIteration):\n                setattr(result.value, \"__context__\", None)\n\n        info.downloading.remove(fp)\n        info.downloaded[fp] = result  # cache result\n        for wad in info.waiting.pop(fp):\n            defer_result(result).chainDeferred(wad)\n\n    # Overridable Interface\n    def media_to_download(self, request, info, *, item=None):\n        \"\"\"Check request before starting download\"\"\"\n        pass\n\n    def get_media_requests(self, item, info):\n        \"\"\"Returns the media requests to download\"\"\"\n        pass\n\n    def media_downloaded(self, response, request, info, *, item=None):\n        \"\"\"Handler for success downloads\"\"\"\n        return response\n\n    def media_failed(self, failure, request, info):\n        \"\"\"Handler for failed downloads\"\"\"\n        return failure\n\n    def item_completed(self, results, item, info):\n        \"\"\"Called per item when all media requests has been processed\"\"\"\n        if self.LOG_FAILED_RESULTS:\n            for ok, value in results:\n                if not ok:\n                    logger.error(\n                        \"%(class)s found errors processing %(item)s\",\n                        {\"class\": self.__class__.__name__, \"item\": item},\n                        exc_info=failure_to_exc_info(value),\n                        extra={\"spider\": info.spider},\n                    )\n        return item\n\n    def file_path(self, request, response=None, info=None, *, item=None):\n        \"\"\"Returns the path where downloaded media should be stored\"\"\"\n        pass\n", "path": "scrapy/pipelines/media.py"}], "after_files": [{"content": "import functools\nimport logging\nfrom collections import defaultdict\n\nfrom twisted.internet.defer import Deferred, DeferredList\nfrom twisted.python.failure import Failure\n\nfrom scrapy.http.request import NO_CALLBACK\nfrom scrapy.settings import Settings\nfrom scrapy.utils.datatypes import SequenceExclude\nfrom scrapy.utils.defer import defer_result, mustbe_deferred\nfrom scrapy.utils.log import failure_to_exc_info\nfrom scrapy.utils.misc import arg_to_iter\n\nlogger = logging.getLogger(__name__)\n\n\ndef _DUMMY_CALLBACK(response):\n    return response\n\n\nclass MediaPipeline:\n    LOG_FAILED_RESULTS = True\n\n    class SpiderInfo:\n        def __init__(self, spider):\n            self.spider = spider\n            self.downloading = set()\n            self.downloaded = {}\n            self.waiting = defaultdict(list)\n\n    def __init__(self, download_func=None, settings=None):\n        self.download_func = download_func\n        self._expects_item = {}\n\n        if isinstance(settings, dict) or settings is None:\n            settings = Settings(settings)\n        resolve = functools.partial(\n            self._key_for_pipe, base_class_name=\"MediaPipeline\", settings=settings\n        )\n        self.allow_redirects = settings.getbool(resolve(\"MEDIA_ALLOW_REDIRECTS\"), False)\n        self._handle_statuses(self.allow_redirects)\n\n    def _handle_statuses(self, allow_redirects):\n        self.handle_httpstatus_list = None\n        if allow_redirects:\n            self.handle_httpstatus_list = SequenceExclude(range(300, 400))\n\n    def _key_for_pipe(self, key, base_class_name=None, settings=None):\n        \"\"\"\n        >>> MediaPipeline()._key_for_pipe(\"IMAGES\")\n        'IMAGES'\n        >>> class MyPipe(MediaPipeline):\n        ...     pass\n        >>> MyPipe()._key_for_pipe(\"IMAGES\", base_class_name=\"MediaPipeline\")\n        'MYPIPE_IMAGES'\n        \"\"\"\n        class_name = self.__class__.__name__\n        formatted_key = f\"{class_name.upper()}_{key}\"\n        if (\n            not base_class_name\n            or class_name == base_class_name\n            or settings\n            and not settings.get(formatted_key)\n        ):\n            return key\n        return formatted_key\n\n    @classmethod\n    def from_crawler(cls, crawler):\n        try:\n            pipe = cls.from_settings(crawler.settings)\n        except AttributeError:\n            pipe = cls()\n        pipe.crawler = crawler\n        pipe._fingerprinter = crawler.request_fingerprinter\n        return pipe\n\n    def open_spider(self, spider):\n        self.spiderinfo = self.SpiderInfo(spider)\n\n    def process_item(self, item, spider):\n        info = self.spiderinfo\n        requests = arg_to_iter(self.get_media_requests(item, info))\n        dlist = [self._process_request(r, info, item) for r in requests]\n        dfd = DeferredList(dlist, consumeErrors=True)\n        return dfd.addCallback(self.item_completed, item, info)\n\n    def _process_request(self, request, info, item):\n        fp = self._fingerprinter.fingerprint(request)\n        if not request.callback or request.callback is NO_CALLBACK:\n            cb = _DUMMY_CALLBACK\n        else:\n            cb = request.callback\n        eb = request.errback\n        request.callback = NO_CALLBACK\n        request.errback = None\n\n        # Return cached result if request was already seen\n        if fp in info.downloaded:\n            return defer_result(info.downloaded[fp]).addCallbacks(cb, eb)\n\n        # Otherwise, wait for result\n        wad = Deferred().addCallbacks(cb, eb)\n        info.waiting[fp].append(wad)\n\n        # Check if request is downloading right now to avoid doing it twice\n        if fp in info.downloading:\n            return wad\n\n        # Download request checking media_to_download hook output first\n        info.downloading.add(fp)\n        dfd = mustbe_deferred(self.media_to_download, request, info, item=item)\n        dfd.addCallback(self._check_media_to_download, request, info, item=item)\n        dfd.addErrback(self._log_exception)\n        dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)\n        return dfd.addBoth(lambda _: wad)  # it must return wad at last\n\n    def _log_exception(self, result):\n        logger.exception(result)\n        return result\n\n    def _modify_media_request(self, request):\n        if self.handle_httpstatus_list:\n            request.meta[\"handle_httpstatus_list\"] = self.handle_httpstatus_list\n        else:\n            request.meta[\"handle_httpstatus_all\"] = True\n\n    def _check_media_to_download(self, result, request, info, item):\n        if result is not None:\n            return result\n        if self.download_func:\n            # this ugly code was left only to support tests. TODO: remove\n            dfd = mustbe_deferred(self.download_func, request, info.spider)\n            dfd.addCallbacks(\n                callback=self.media_downloaded,\n                callbackArgs=(request, info),\n                callbackKeywords={\"item\": item},\n                errback=self.media_failed,\n                errbackArgs=(request, info),\n            )\n        else:\n            self._modify_media_request(request)\n            dfd = self.crawler.engine.download(request)\n            dfd.addCallbacks(\n                callback=self.media_downloaded,\n                callbackArgs=(request, info),\n                callbackKeywords={\"item\": item},\n                errback=self.media_failed,\n                errbackArgs=(request, info),\n            )\n        return dfd\n\n    def _cache_result_and_execute_waiters(self, result, fp, info):\n        if isinstance(result, Failure):\n            # minimize cached information for failure\n            result.cleanFailure()\n            result.frames = []\n            result.stack = None\n\n            # This code fixes a memory leak by avoiding to keep references to\n            # the Request and Response objects on the Media Pipeline cache.\n            #\n            # What happens when the media_downloaded callback raises an\n            # exception, for example a FileException('download-error') when\n            # the Response status code is not 200 OK, is that the original\n            # StopIteration exception (which in turn contains the failed\n            # Response and by extension, the original Request) gets encapsulated\n            # within the FileException context.\n            #\n            # Originally, Scrapy was using twisted.internet.defer.returnValue\n            # inside functions decorated with twisted.internet.defer.inlineCallbacks,\n            # encapsulating the returned Response in a _DefGen_Return exception\n            # instead of a StopIteration.\n            #\n            # To avoid keeping references to the Response and therefore Request\n            # objects on the Media Pipeline cache, we should wipe the context of\n            # the encapsulated exception when it is a StopIteration instance\n            #\n            # This problem does not occur in Python 2.7 since we don't have\n            # Exception Chaining (https://www.python.org/dev/peps/pep-3134/).\n            context = getattr(result.value, \"__context__\", None)\n            if isinstance(context, StopIteration):\n                setattr(result.value, \"__context__\", None)\n\n        info.downloading.remove(fp)\n        info.downloaded[fp] = result  # cache result\n        for wad in info.waiting.pop(fp):\n            defer_result(result).chainDeferred(wad)\n\n    # Overridable Interface\n    def media_to_download(self, request, info, *, item=None):\n        \"\"\"Check request before starting download\"\"\"\n        pass\n\n    def get_media_requests(self, item, info):\n        \"\"\"Returns the media requests to download\"\"\"\n        pass\n\n    def media_downloaded(self, response, request, info, *, item=None):\n        \"\"\"Handler for success downloads\"\"\"\n        return response\n\n    def media_failed(self, failure, request, info):\n        \"\"\"Handler for failed downloads\"\"\"\n        return failure\n\n    def item_completed(self, results, item, info):\n        \"\"\"Called per item when all media requests has been processed\"\"\"\n        if self.LOG_FAILED_RESULTS:\n            for ok, value in results:\n                if not ok:\n                    logger.error(\n                        \"%(class)s found errors processing %(item)s\",\n                        {\"class\": self.__class__.__name__, \"item\": item},\n                        exc_info=failure_to_exc_info(value),\n                        extra={\"spider\": info.spider},\n                    )\n        return item\n\n    def file_path(self, request, response=None, info=None, *, item=None):\n        \"\"\"Returns the path where downloaded media should be stored\"\"\"\n        pass\n", "path": "scrapy/pipelines/media.py"}]}