We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
scrapy parse -h throws error
### Description
running `scrapy parse -h` from inside of a project throws an error.

### Steps to Reproduce
1.  `scrapy startproject example`
2.  `cd example`
3.  `scrapy parse -h`

**Expected behavior:**  It should show the help message output for the `parse` command
**Actual behavior:**  Throws error and displays traceback information
**Reproduces how often:**  Every time I tried it.

### Versions
v2.6.1

### Additional context

I identified the issue and will submit a PR shortly. 

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/commands/parse.py`
Content:
```
1 import json
2 import logging
3 from typing import Dict
4 
5 from itemadapter import is_item, ItemAdapter
6 from w3lib.url import is_url
7 
8 from scrapy.commands import BaseRunSpiderCommand
9 from scrapy.http import Request
10 from scrapy.utils import display
11 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
12 from scrapy.exceptions import UsageError
13 
14 
15 logger = logging.getLogger(__name__)
16 
17 
18 class Command(BaseRunSpiderCommand):
19     requires_project = True
20 
21     spider = None
22     items: Dict[int, list] = {}
23     requests: Dict[int, list] = {}
24 
25     first_response = None
26 
27     def syntax(self):
28         return "[options] <url>"
29 
30     def short_desc(self):
31         return "Parse URL (using its spider) and print the results"
32 
33     def add_options(self, parser):
34         BaseRunSpiderCommand.add_options(self, parser)
35         parser.add_argument("--spider", dest="spider", default=None,
36                             help="use this spider without looking for one")
37         parser.add_argument("--pipelines", action="store_true",
38                             help="process items through pipelines")
39         parser.add_argument("--nolinks", dest="nolinks", action="store_true",
40                             help="don't show links to follow (extracted requests)")
41         parser.add_argument("--noitems", dest="noitems", action="store_true",
42                             help="don't show scraped items")
43         parser.add_argument("--nocolour", dest="nocolour", action="store_true",
44                             help="avoid using pygments to colorize the output")
45         parser.add_argument("-r", "--rules", dest="rules", action="store_true",
46                             help="use CrawlSpider rules to discover the callback")
47         parser.add_argument("-c", "--callback", dest="callback",
48                             help="use this callback for parsing, instead looking for a callback")
49         parser.add_argument("-m", "--meta", dest="meta",
50                             help="inject extra meta into the Request, it must be a valid raw json string")
51         parser.add_argument("--cbkwargs", dest="cbkwargs",
52                             help="inject extra callback kwargs into the Request, it must be a valid raw json string")
53         parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
54                             help="maximum depth for parsing requests [default: %default]")
55         parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
56                             help="print each depth level one by one")
57 
58     @property
59     def max_level(self):
60         max_items, max_requests = 0, 0
61         if self.items:
62             max_items = max(self.items)
63         if self.requests:
64             max_requests = max(self.requests)
65         return max(max_items, max_requests)
66 
67     def add_items(self, lvl, new_items):
68         old_items = self.items.get(lvl, [])
69         self.items[lvl] = old_items + new_items
70 
71     def add_requests(self, lvl, new_reqs):
72         old_reqs = self.requests.get(lvl, [])
73         self.requests[lvl] = old_reqs + new_reqs
74 
75     def print_items(self, lvl=None, colour=True):
76         if lvl is None:
77             items = [item for lst in self.items.values() for item in lst]
78         else:
79             items = self.items.get(lvl, [])
80 
81         print("# Scraped Items ", "-" * 60)
82         display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
83 
84     def print_requests(self, lvl=None, colour=True):
85         if lvl is None:
86             if self.requests:
87                 requests = self.requests[max(self.requests)]
88             else:
89                 requests = []
90         else:
91             requests = self.requests.get(lvl, [])
92 
93         print("# Requests ", "-" * 65)
94         display.pprint(requests, colorize=colour)
95 
96     def print_results(self, opts):
97         colour = not opts.nocolour
98 
99         if opts.verbose:
100             for level in range(1, self.max_level + 1):
101                 print(f'\n>>> DEPTH LEVEL: {level} <<<')
102                 if not opts.noitems:
103                     self.print_items(level, colour)
104                 if not opts.nolinks:
105                     self.print_requests(level, colour)
106         else:
107             print(f'\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')
108             if not opts.noitems:
109                 self.print_items(colour=colour)
110             if not opts.nolinks:
111                 self.print_requests(colour=colour)
112 
113     def run_callback(self, response, callback, cb_kwargs=None):
114         cb_kwargs = cb_kwargs or {}
115         items, requests = [], []
116 
117         for x in iterate_spider_output(callback(response, **cb_kwargs)):
118             if is_item(x):
119                 items.append(x)
120             elif isinstance(x, Request):
121                 requests.append(x)
122         return items, requests
123 
124     def get_callback_from_rules(self, spider, response):
125         if getattr(spider, 'rules', None):
126             for rule in spider.rules:
127                 if rule.link_extractor.matches(response.url):
128                     return rule.callback or "parse"
129         else:
130             logger.error('No CrawlSpider rules found in spider %(spider)r, '
131                          'please specify a callback to use for parsing',
132                          {'spider': spider.name})
133 
134     def set_spidercls(self, url, opts):
135         spider_loader = self.crawler_process.spider_loader
136         if opts.spider:
137             try:
138                 self.spidercls = spider_loader.load(opts.spider)
139             except KeyError:
140                 logger.error('Unable to find spider: %(spider)s',
141                              {'spider': opts.spider})
142         else:
143             self.spidercls = spidercls_for_request(spider_loader, Request(url))
144             if not self.spidercls:
145                 logger.error('Unable to find spider for: %(url)s', {'url': url})
146 
147         def _start_requests(spider):
148             yield self.prepare_request(spider, Request(url), opts)
149         self.spidercls.start_requests = _start_requests
150 
151     def start_parsing(self, url, opts):
152         self.crawler_process.crawl(self.spidercls, **opts.spargs)
153         self.pcrawler = list(self.crawler_process.crawlers)[0]
154         self.crawler_process.start()
155 
156         if not self.first_response:
157             logger.error('No response downloaded for: %(url)s',
158                          {'url': url})
159 
160     def prepare_request(self, spider, request, opts):
161         def callback(response, **cb_kwargs):
162             # memorize first request
163             if not self.first_response:
164                 self.first_response = response
165 
166             # determine real callback
167             cb = response.meta['_callback']
168             if not cb:
169                 if opts.callback:
170                     cb = opts.callback
171                 elif opts.rules and self.first_response == response:
172                     cb = self.get_callback_from_rules(spider, response)
173 
174                     if not cb:
175                         logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
176                                      {'url': response.url, 'spider': spider.name})
177                         return
178                 else:
179                     cb = 'parse'
180 
181             if not callable(cb):
182                 cb_method = getattr(spider, cb, None)
183                 if callable(cb_method):
184                     cb = cb_method
185                 else:
186                     logger.error('Cannot find callback %(callback)r in spider: %(spider)s',
187                                  {'callback': cb, 'spider': spider.name})
188                     return
189 
190             # parse items and requests
191             depth = response.meta['_depth']
192 
193             items, requests = self.run_callback(response, cb, cb_kwargs)
194             if opts.pipelines:
195                 itemproc = self.pcrawler.engine.scraper.itemproc
196                 for item in items:
197                     itemproc.process_item(item, spider)
198             self.add_items(depth, items)
199             self.add_requests(depth, requests)
200 
201             scraped_data = items if opts.output else []
202             if depth < opts.depth:
203                 for req in requests:
204                     req.meta['_depth'] = depth + 1
205                     req.meta['_callback'] = req.callback
206                     req.callback = callback
207                 scraped_data += requests
208 
209             return scraped_data
210 
211         # update request meta if any extra meta was passed through the --meta/-m opts.
212         if opts.meta:
213             request.meta.update(opts.meta)
214 
215         # update cb_kwargs if any extra values were was passed through the --cbkwargs option.
216         if opts.cbkwargs:
217             request.cb_kwargs.update(opts.cbkwargs)
218 
219         request.meta['_depth'] = 1
220         request.meta['_callback'] = request.callback
221         request.callback = callback
222         return request
223 
224     def process_options(self, args, opts):
225         BaseRunSpiderCommand.process_options(self, args, opts)
226 
227         self.process_request_meta(opts)
228         self.process_request_cb_kwargs(opts)
229 
230     def process_request_meta(self, opts):
231         if opts.meta:
232             try:
233                 opts.meta = json.loads(opts.meta)
234             except ValueError:
235                 raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
236                                  "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
237 
238     def process_request_cb_kwargs(self, opts):
239         if opts.cbkwargs:
240             try:
241                 opts.cbkwargs = json.loads(opts.cbkwargs)
242             except ValueError:
243                 raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
244                                  "Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False)
245 
246     def run(self, args, opts):
247         # parse arguments
248         if not len(args) == 1 or not is_url(args[0]):
249             raise UsageError()
250         else:
251             url = args[0]
252 
253         # prepare spidercls
254         self.set_spidercls(url, opts)
255 
256         if self.spidercls and opts.depth > 0:
257             self.start_parsing(url, opts)
258             self.print_results(opts)
259
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -51,7 +51,7 @@
         parser.add_argument("--cbkwargs", dest="cbkwargs",
                             help="inject extra callback kwargs into the Request, it must be a valid raw json string")
         parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
-                            help="maximum depth for parsing requests [default: %default]")
+                            help="maximum depth for parsing requests [default: %(default)s]")
         parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                             help="print each depth level one by one")

{"golden_diff": "diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py\n--- a/scrapy/commands/parse.py\n+++ b/scrapy/commands/parse.py\n@@ -51,7 +51,7 @@\n         parser.add_argument(\"--cbkwargs\", dest=\"cbkwargs\",\n                             help=\"inject extra callback kwargs into the Request, it must be a valid raw json string\")\n         parser.add_argument(\"-d\", \"--depth\", dest=\"depth\", type=int, default=1,\n-                            help=\"maximum depth for parsing requests [default: %default]\")\n+                            help=\"maximum depth for parsing requests [default: %(default)s]\")\n         parser.add_argument(\"-v\", \"--verbose\", dest=\"verbose\", action=\"store_true\",\n                             help=\"print each depth level one by one\")\n", "issue": "scrapy parse -h throws error\n### Description\r\nrunning `scrapy parse -h` from inside of a project throws an error.\r\n\r\n### Steps to Reproduce\r\n1.  `scrapy startproject example`\r\n2.  `cd example`\r\n3.  `scrapy parse -h`\r\n\r\n**Expected behavior:**  It should show the help message output for the `parse` command\r\n**Actual behavior:**  Throws error and displays traceback information\r\n**Reproduces how often:**  Every time I tried it.\r\n\r\n### Versions\r\nv2.6.1\r\n\r\n### Additional context\r\n\r\nI identified the issue and will submit a PR shortly. \n", "before_files": [{"content": "import json\nimport logging\nfrom typing import Dict\n\nfrom itemadapter import is_item, ItemAdapter\nfrom w3lib.url import is_url\n\nfrom scrapy.commands import BaseRunSpiderCommand\nfrom scrapy.http import Request\nfrom scrapy.utils import display\nfrom scrapy.utils.spider import iterate_spider_output, spidercls_for_request\nfrom scrapy.exceptions import UsageError\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass Command(BaseRunSpiderCommand):\n    requires_project = True\n\n    spider = None\n    items: Dict[int, list] = {}\n    requests: Dict[int, list] = {}\n\n    first_response = None\n\n    def syntax(self):\n        return \"[options] <url>\"\n\n    def short_desc(self):\n        return \"Parse URL (using its spider) and print the results\"\n\n    def add_options(self, parser):\n        BaseRunSpiderCommand.add_options(self, parser)\n        parser.add_argument(\"--spider\", dest=\"spider\", default=None,\n                            help=\"use this spider without looking for one\")\n        parser.add_argument(\"--pipelines\", action=\"store_true\",\n                            help=\"process items through pipelines\")\n        parser.add_argument(\"--nolinks\", dest=\"nolinks\", action=\"store_true\",\n                            help=\"don't show links to follow (extracted requests)\")\n        parser.add_argument(\"--noitems\", dest=\"noitems\", action=\"store_true\",\n                            help=\"don't show scraped items\")\n        parser.add_argument(\"--nocolour\", dest=\"nocolour\", action=\"store_true\",\n                            help=\"avoid using pygments to colorize the output\")\n        parser.add_argument(\"-r\", \"--rules\", dest=\"rules\", action=\"store_true\",\n                            help=\"use CrawlSpider rules to discover the callback\")\n        parser.add_argument(\"-c\", \"--callback\", dest=\"callback\",\n                            help=\"use this callback for parsing, instead looking for a callback\")\n        parser.add_argument(\"-m\", \"--meta\", dest=\"meta\",\n                            help=\"inject extra meta into the Request, it must be a valid raw json string\")\n        parser.add_argument(\"--cbkwargs\", dest=\"cbkwargs\",\n                            help=\"inject extra callback kwargs into the Request, it must be a valid raw json string\")\n        parser.add_argument(\"-d\", \"--depth\", dest=\"depth\", type=int, default=1,\n                            help=\"maximum depth for parsing requests [default: %default]\")\n        parser.add_argument(\"-v\", \"--verbose\", dest=\"verbose\", action=\"store_true\",\n                            help=\"print each depth level one by one\")\n\n    @property\n    def max_level(self):\n        max_items, max_requests = 0, 0\n        if self.items:\n            max_items = max(self.items)\n        if self.requests:\n            max_requests = max(self.requests)\n        return max(max_items, max_requests)\n\n    def add_items(self, lvl, new_items):\n        old_items = self.items.get(lvl, [])\n        self.items[lvl] = old_items + new_items\n\n    def add_requests(self, lvl, new_reqs):\n        old_reqs = self.requests.get(lvl, [])\n        self.requests[lvl] = old_reqs + new_reqs\n\n    def print_items(self, lvl=None, colour=True):\n        if lvl is None:\n            items = [item for lst in self.items.values() for item in lst]\n        else:\n            items = self.items.get(lvl, [])\n\n        print(\"# Scraped Items \", \"-\" * 60)\n        display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)\n\n    def print_requests(self, lvl=None, colour=True):\n        if lvl is None:\n            if self.requests:\n                requests = self.requests[max(self.requests)]\n            else:\n                requests = []\n        else:\n            requests = self.requests.get(lvl, [])\n\n        print(\"# Requests \", \"-\" * 65)\n        display.pprint(requests, colorize=colour)\n\n    def print_results(self, opts):\n        colour = not opts.nocolour\n\n        if opts.verbose:\n            for level in range(1, self.max_level + 1):\n                print(f'\\n>>> DEPTH LEVEL: {level} <<<')\n                if not opts.noitems:\n                    self.print_items(level, colour)\n                if not opts.nolinks:\n                    self.print_requests(level, colour)\n        else:\n            print(f'\\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')\n            if not opts.noitems:\n                self.print_items(colour=colour)\n            if not opts.nolinks:\n                self.print_requests(colour=colour)\n\n    def run_callback(self, response, callback, cb_kwargs=None):\n        cb_kwargs = cb_kwargs or {}\n        items, requests = [], []\n\n        for x in iterate_spider_output(callback(response, **cb_kwargs)):\n            if is_item(x):\n                items.append(x)\n            elif isinstance(x, Request):\n                requests.append(x)\n        return items, requests\n\n    def get_callback_from_rules(self, spider, response):\n        if getattr(spider, 'rules', None):\n            for rule in spider.rules:\n                if rule.link_extractor.matches(response.url):\n                    return rule.callback or \"parse\"\n        else:\n            logger.error('No CrawlSpider rules found in spider %(spider)r, '\n                         'please specify a callback to use for parsing',\n                         {'spider': spider.name})\n\n    def set_spidercls(self, url, opts):\n        spider_loader = self.crawler_process.spider_loader\n        if opts.spider:\n            try:\n                self.spidercls = spider_loader.load(opts.spider)\n            except KeyError:\n                logger.error('Unable to find spider: %(spider)s',\n                             {'spider': opts.spider})\n        else:\n            self.spidercls = spidercls_for_request(spider_loader, Request(url))\n            if not self.spidercls:\n                logger.error('Unable to find spider for: %(url)s', {'url': url})\n\n        def _start_requests(spider):\n            yield self.prepare_request(spider, Request(url), opts)\n        self.spidercls.start_requests = _start_requests\n\n    def start_parsing(self, url, opts):\n        self.crawler_process.crawl(self.spidercls, **opts.spargs)\n        self.pcrawler = list(self.crawler_process.crawlers)[0]\n        self.crawler_process.start()\n\n        if not self.first_response:\n            logger.error('No response downloaded for: %(url)s',\n                         {'url': url})\n\n    def prepare_request(self, spider, request, opts):\n        def callback(response, **cb_kwargs):\n            # memorize first request\n            if not self.first_response:\n                self.first_response = response\n\n            # determine real callback\n            cb = response.meta['_callback']\n            if not cb:\n                if opts.callback:\n                    cb = opts.callback\n                elif opts.rules and self.first_response == response:\n                    cb = self.get_callback_from_rules(spider, response)\n\n                    if not cb:\n                        logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',\n                                     {'url': response.url, 'spider': spider.name})\n                        return\n                else:\n                    cb = 'parse'\n\n            if not callable(cb):\n                cb_method = getattr(spider, cb, None)\n                if callable(cb_method):\n                    cb = cb_method\n                else:\n                    logger.error('Cannot find callback %(callback)r in spider: %(spider)s',\n                                 {'callback': cb, 'spider': spider.name})\n                    return\n\n            # parse items and requests\n            depth = response.meta['_depth']\n\n            items, requests = self.run_callback(response, cb, cb_kwargs)\n            if opts.pipelines:\n                itemproc = self.pcrawler.engine.scraper.itemproc\n                for item in items:\n                    itemproc.process_item(item, spider)\n            self.add_items(depth, items)\n            self.add_requests(depth, requests)\n\n            scraped_data = items if opts.output else []\n            if depth < opts.depth:\n                for req in requests:\n                    req.meta['_depth'] = depth + 1\n                    req.meta['_callback'] = req.callback\n                    req.callback = callback\n                scraped_data += requests\n\n            return scraped_data\n\n        # update request meta if any extra meta was passed through the --meta/-m opts.\n        if opts.meta:\n            request.meta.update(opts.meta)\n\n        # update cb_kwargs if any extra values were was passed through the --cbkwargs option.\n        if opts.cbkwargs:\n            request.cb_kwargs.update(opts.cbkwargs)\n\n        request.meta['_depth'] = 1\n        request.meta['_callback'] = request.callback\n        request.callback = callback\n        return request\n\n    def process_options(self, args, opts):\n        BaseRunSpiderCommand.process_options(self, args, opts)\n\n        self.process_request_meta(opts)\n        self.process_request_cb_kwargs(opts)\n\n    def process_request_meta(self, opts):\n        if opts.meta:\n            try:\n                opts.meta = json.loads(opts.meta)\n            except ValueError:\n                raise UsageError(\"Invalid -m/--meta value, pass a valid json string to -m or --meta. \"\n                                 \"Example: --meta='{\\\"foo\\\" : \\\"bar\\\"}'\", print_help=False)\n\n    def process_request_cb_kwargs(self, opts):\n        if opts.cbkwargs:\n            try:\n                opts.cbkwargs = json.loads(opts.cbkwargs)\n            except ValueError:\n                raise UsageError(\"Invalid --cbkwargs value, pass a valid json string to --cbkwargs. \"\n                                 \"Example: --cbkwargs='{\\\"foo\\\" : \\\"bar\\\"}'\", print_help=False)\n\n    def run(self, args, opts):\n        # parse arguments\n        if not len(args) == 1 or not is_url(args[0]):\n            raise UsageError()\n        else:\n            url = args[0]\n\n        # prepare spidercls\n        self.set_spidercls(url, opts)\n\n        if self.spidercls and opts.depth > 0:\n            self.start_parsing(url, opts)\n            self.print_results(opts)\n", "path": "scrapy/commands/parse.py"}], "after_files": [{"content": "import json\nimport logging\nfrom typing import Dict\n\nfrom itemadapter import is_item, ItemAdapter\nfrom w3lib.url import is_url\n\nfrom scrapy.commands import BaseRunSpiderCommand\nfrom scrapy.http import Request\nfrom scrapy.utils import display\nfrom scrapy.utils.spider import iterate_spider_output, spidercls_for_request\nfrom scrapy.exceptions import UsageError\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass Command(BaseRunSpiderCommand):\n    requires_project = True\n\n    spider = None\n    items: Dict[int, list] = {}\n    requests: Dict[int, list] = {}\n\n    first_response = None\n\n    def syntax(self):\n        return \"[options] <url>\"\n\n    def short_desc(self):\n        return \"Parse URL (using its spider) and print the results\"\n\n    def add_options(self, parser):\n        BaseRunSpiderCommand.add_options(self, parser)\n        parser.add_argument(\"--spider\", dest=\"spider\", default=None,\n                            help=\"use this spider without looking for one\")\n        parser.add_argument(\"--pipelines\", action=\"store_true\",\n                            help=\"process items through pipelines\")\n        parser.add_argument(\"--nolinks\", dest=\"nolinks\", action=\"store_true\",\n                            help=\"don't show links to follow (extracted requests)\")\n        parser.add_argument(\"--noitems\", dest=\"noitems\", action=\"store_true\",\n                            help=\"don't show scraped items\")\n        parser.add_argument(\"--nocolour\", dest=\"nocolour\", action=\"store_true\",\n                            help=\"avoid using pygments to colorize the output\")\n        parser.add_argument(\"-r\", \"--rules\", dest=\"rules\", action=\"store_true\",\n                            help=\"use CrawlSpider rules to discover the callback\")\n        parser.add_argument(\"-c\", \"--callback\", dest=\"callback\",\n                            help=\"use this callback for parsing, instead looking for a callback\")\n        parser.add_argument(\"-m\", \"--meta\", dest=\"meta\",\n                            help=\"inject extra meta into the Request, it must be a valid raw json string\")\n        parser.add_argument(\"--cbkwargs\", dest=\"cbkwargs\",\n                            help=\"inject extra callback kwargs into the Request, it must be a valid raw json string\")\n        parser.add_argument(\"-d\", \"--depth\", dest=\"depth\", type=int, default=1,\n                            help=\"maximum depth for parsing requests [default: %(default)s]\")\n        parser.add_argument(\"-v\", \"--verbose\", dest=\"verbose\", action=\"store_true\",\n                            help=\"print each depth level one by one\")\n\n    @property\n    def max_level(self):\n        max_items, max_requests = 0, 0\n        if self.items:\n            max_items = max(self.items)\n        if self.requests:\n            max_requests = max(self.requests)\n        return max(max_items, max_requests)\n\n    def add_items(self, lvl, new_items):\n        old_items = self.items.get(lvl, [])\n        self.items[lvl] = old_items + new_items\n\n    def add_requests(self, lvl, new_reqs):\n        old_reqs = self.requests.get(lvl, [])\n        self.requests[lvl] = old_reqs + new_reqs\n\n    def print_items(self, lvl=None, colour=True):\n        if lvl is None:\n            items = [item for lst in self.items.values() for item in lst]\n        else:\n            items = self.items.get(lvl, [])\n\n        print(\"# Scraped Items \", \"-\" * 60)\n        display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)\n\n    def print_requests(self, lvl=None, colour=True):\n        if lvl is None:\n            if self.requests:\n                requests = self.requests[max(self.requests)]\n            else:\n                requests = []\n        else:\n            requests = self.requests.get(lvl, [])\n\n        print(\"# Requests \", \"-\" * 65)\n        display.pprint(requests, colorize=colour)\n\n    def print_results(self, opts):\n        colour = not opts.nocolour\n\n        if opts.verbose:\n            for level in range(1, self.max_level + 1):\n                print(f'\\n>>> DEPTH LEVEL: {level} <<<')\n                if not opts.noitems:\n                    self.print_items(level, colour)\n                if not opts.nolinks:\n                    self.print_requests(level, colour)\n        else:\n            print(f'\\n>>> STATUS DEPTH LEVEL {self.max_level} <<<')\n            if not opts.noitems:\n                self.print_items(colour=colour)\n            if not opts.nolinks:\n                self.print_requests(colour=colour)\n\n    def run_callback(self, response, callback, cb_kwargs=None):\n        cb_kwargs = cb_kwargs or {}\n        items, requests = [], []\n\n        for x in iterate_spider_output(callback(response, **cb_kwargs)):\n            if is_item(x):\n                items.append(x)\n            elif isinstance(x, Request):\n                requests.append(x)\n        return items, requests\n\n    def get_callback_from_rules(self, spider, response):\n        if getattr(spider, 'rules', None):\n            for rule in spider.rules:\n                if rule.link_extractor.matches(response.url):\n                    return rule.callback or \"parse\"\n        else:\n            logger.error('No CrawlSpider rules found in spider %(spider)r, '\n                         'please specify a callback to use for parsing',\n                         {'spider': spider.name})\n\n    def set_spidercls(self, url, opts):\n        spider_loader = self.crawler_process.spider_loader\n        if opts.spider:\n            try:\n                self.spidercls = spider_loader.load(opts.spider)\n            except KeyError:\n                logger.error('Unable to find spider: %(spider)s',\n                             {'spider': opts.spider})\n        else:\n            self.spidercls = spidercls_for_request(spider_loader, Request(url))\n            if not self.spidercls:\n                logger.error('Unable to find spider for: %(url)s', {'url': url})\n\n        def _start_requests(spider):\n            yield self.prepare_request(spider, Request(url), opts)\n        self.spidercls.start_requests = _start_requests\n\n    def start_parsing(self, url, opts):\n        self.crawler_process.crawl(self.spidercls, **opts.spargs)\n        self.pcrawler = list(self.crawler_process.crawlers)[0]\n        self.crawler_process.start()\n\n        if not self.first_response:\n            logger.error('No response downloaded for: %(url)s',\n                         {'url': url})\n\n    def prepare_request(self, spider, request, opts):\n        def callback(response, **cb_kwargs):\n            # memorize first request\n            if not self.first_response:\n                self.first_response = response\n\n            # determine real callback\n            cb = response.meta['_callback']\n            if not cb:\n                if opts.callback:\n                    cb = opts.callback\n                elif opts.rules and self.first_response == response:\n                    cb = self.get_callback_from_rules(spider, response)\n\n                    if not cb:\n                        logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',\n                                     {'url': response.url, 'spider': spider.name})\n                        return\n                else:\n                    cb = 'parse'\n\n            if not callable(cb):\n                cb_method = getattr(spider, cb, None)\n                if callable(cb_method):\n                    cb = cb_method\n                else:\n                    logger.error('Cannot find callback %(callback)r in spider: %(spider)s',\n                                 {'callback': cb, 'spider': spider.name})\n                    return\n\n            # parse items and requests\n            depth = response.meta['_depth']\n\n            items, requests = self.run_callback(response, cb, cb_kwargs)\n            if opts.pipelines:\n                itemproc = self.pcrawler.engine.scraper.itemproc\n                for item in items:\n                    itemproc.process_item(item, spider)\n            self.add_items(depth, items)\n            self.add_requests(depth, requests)\n\n            scraped_data = items if opts.output else []\n            if depth < opts.depth:\n                for req in requests:\n                    req.meta['_depth'] = depth + 1\n                    req.meta['_callback'] = req.callback\n                    req.callback = callback\n                scraped_data += requests\n\n            return scraped_data\n\n        # update request meta if any extra meta was passed through the --meta/-m opts.\n        if opts.meta:\n            request.meta.update(opts.meta)\n\n        # update cb_kwargs if any extra values were was passed through the --cbkwargs option.\n        if opts.cbkwargs:\n            request.cb_kwargs.update(opts.cbkwargs)\n\n        request.meta['_depth'] = 1\n        request.meta['_callback'] = request.callback\n        request.callback = callback\n        return request\n\n    def process_options(self, args, opts):\n        BaseRunSpiderCommand.process_options(self, args, opts)\n\n        self.process_request_meta(opts)\n        self.process_request_cb_kwargs(opts)\n\n    def process_request_meta(self, opts):\n        if opts.meta:\n            try:\n                opts.meta = json.loads(opts.meta)\n            except ValueError:\n                raise UsageError(\"Invalid -m/--meta value, pass a valid json string to -m or --meta. \"\n                                 \"Example: --meta='{\\\"foo\\\" : \\\"bar\\\"}'\", print_help=False)\n\n    def process_request_cb_kwargs(self, opts):\n        if opts.cbkwargs:\n            try:\n                opts.cbkwargs = json.loads(opts.cbkwargs)\n            except ValueError:\n                raise UsageError(\"Invalid --cbkwargs value, pass a valid json string to --cbkwargs. \"\n                                 \"Example: --cbkwargs='{\\\"foo\\\" : \\\"bar\\\"}'\", print_help=False)\n\n    def run(self, args, opts):\n        # parse arguments\n        if not len(args) == 1 or not is_url(args[0]):\n            raise UsageError()\n        else:\n            url = args[0]\n\n        # prepare spidercls\n        self.set_spidercls(url, opts)\n\n        if self.spidercls and opts.depth > 0:\n            self.start_parsing(url, opts)\n            self.print_results(opts)\n", "path": "scrapy/commands/parse.py"}]}