We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Set environment variable when running scrapy check
Sometimes it is nice to be able to enable/disable functionality, e.g. calculating things in settings.py when just checking spider contracts instead of running a crawl. I therefor propose setting an environment variable like `SCRAPY_CHECK` when using the check command.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/utils/misc.py`
Content:
```
1 """Helper functions which don't fit anywhere else"""
2 import re
3 import hashlib
4 from importlib import import_module
5 from pkgutil import iter_modules
6 
7 import six
8 from w3lib.html import replace_entities
9 
10 from scrapy.utils.python import flatten, to_unicode
11 from scrapy.item import BaseItem
12 
13 
14 _ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes
15 
16 
17 def arg_to_iter(arg):
18     """Convert an argument to an iterable. The argument can be a None, single
19     value, or an iterable.
20 
21     Exception: if arg is a dict, [arg] will be returned
22     """
23     if arg is None:
24         return []
25     elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):
26         return arg
27     else:
28         return [arg]
29 
30 
31 def load_object(path):
32     """Load an object given its absolute object path, and return it.
33 
34     object can be a class, function, variable or an instance.
35     path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'
36     """
37 
38     try:
39         dot = path.rindex('.')
40     except ValueError:
41         raise ValueError("Error loading object '%s': not a full path" % path)
42 
43     module, name = path[:dot], path[dot+1:]
44     mod = import_module(module)
45 
46     try:
47         obj = getattr(mod, name)
48     except AttributeError:
49         raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name))
50 
51     return obj
52 
53 
54 def walk_modules(path):
55     """Loads a module and all its submodules from the given module path and
56     returns them. If *any* module throws an exception while importing, that
57     exception is thrown back.
58 
59     For example: walk_modules('scrapy.utils')
60     """
61 
62     mods = []
63     mod = import_module(path)
64     mods.append(mod)
65     if hasattr(mod, '__path__'):
66         for _, subpath, ispkg in iter_modules(mod.__path__):
67             fullpath = path + '.' + subpath
68             if ispkg:
69                 mods += walk_modules(fullpath)
70             else:
71                 submod = import_module(fullpath)
72                 mods.append(submod)
73     return mods
74 
75 
76 def extract_regex(regex, text, encoding='utf-8'):
77     """Extract a list of unicode strings from the given text/encoding using the following policies:
78 
79     * if the regex contains a named group called "extract" that will be returned
80     * if the regex contains multiple numbered groups, all those will be returned (flattened)
81     * if the regex doesn't contain any group the entire regex matching is returned
82     """
83 
84     if isinstance(regex, six.string_types):
85         regex = re.compile(regex, re.UNICODE)
86 
87     try:
88         strings = [regex.search(text).group('extract')]   # named group
89     except Exception:
90         strings = regex.findall(text)    # full regex or numbered groups
91     strings = flatten(strings)
92 
93     if isinstance(text, six.text_type):
94         return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
95     else:
96         return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
97                 for s in strings]
98 
99 
100 def md5sum(file):
101     """Calculate the md5 checksum of a file-like object without reading its
102     whole content in memory.
103 
104     >>> from io import BytesIO
105     >>> md5sum(BytesIO(b'file content to hash'))
106     '784406af91dd5a54fbb9c84c2236595a'
107     """
108     m = hashlib.md5()
109     while True:
110         d = file.read(8096)
111         if not d:
112             break
113         m.update(d)
114     return m.hexdigest()
115 
116 
117 def rel_has_nofollow(rel):
118     """Return True if link rel attribute has nofollow type"""
119     return rel is not None and 'nofollow' in rel.split()
120 
121 
122 def create_instance(objcls, settings, crawler, *args, **kwargs):
123     """Construct a class instance using its ``from_crawler`` or
124     ``from_settings`` constructors, if available.
125 
126     At least one of ``settings`` and ``crawler`` needs to be different from
127     ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
128     If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
129     tried.
130 
131     ``*args`` and ``**kwargs`` are forwarded to the constructors.
132 
133     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
134     """
135     if settings is None:
136         if crawler is None:
137             raise ValueError("Specifiy at least one of settings and crawler.")
138         settings = crawler.settings
139     if crawler and hasattr(objcls, 'from_crawler'):
140         return objcls.from_crawler(crawler, *args, **kwargs)
141     elif hasattr(objcls, 'from_settings'):
142         return objcls.from_settings(settings, *args, **kwargs)
143     else:
144         return objcls(*args, **kwargs)
145
```

Path: `scrapy/commands/check.py`
Content:
```
1 from __future__ import print_function
2 import time
3 import sys
4 from collections import defaultdict
5 from unittest import TextTestRunner, TextTestResult as _TextTestResult
6 
7 from scrapy.commands import ScrapyCommand
8 from scrapy.contracts import ContractsManager
9 from scrapy.utils.misc import load_object
10 from scrapy.utils.conf import build_component_list
11 
12 
13 class TextTestResult(_TextTestResult):
14     def printSummary(self, start, stop):
15         write = self.stream.write
16         writeln = self.stream.writeln
17 
18         run = self.testsRun
19         plural = "s" if run != 1 else ""
20 
21         writeln(self.separator2)
22         writeln("Ran %d contract%s in %.3fs" % (run, plural, stop - start))
23         writeln()
24 
25         infos = []
26         if not self.wasSuccessful():
27             write("FAILED")
28             failed, errored = map(len, (self.failures, self.errors))
29             if failed:
30                 infos.append("failures=%d" % failed)
31             if errored:
32                 infos.append("errors=%d" % errored)
33         else:
34             write("OK")
35 
36         if infos:
37             writeln(" (%s)" % (", ".join(infos),))
38         else:
39             write("\n")
40 
41 
42 class Command(ScrapyCommand):
43     requires_project = True
44     default_settings = {'LOG_ENABLED': False}
45 
46     def syntax(self):
47         return "[options] <spider>"
48 
49     def short_desc(self):
50         return "Check spider contracts"
51 
52     def add_options(self, parser):
53         ScrapyCommand.add_options(self, parser)
54         parser.add_option("-l", "--list", dest="list", action="store_true",
55                           help="only list contracts, without checking them")
56         parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
57                           help="print contract tests for all spiders")
58 
59     def run(self, args, opts):
60         # load contracts
61         contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS'))
62         conman = ContractsManager(load_object(c) for c in contracts)
63         runner = TextTestRunner(verbosity=2 if opts.verbose else 1)
64         result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)
65 
66         # contract requests
67         contract_reqs = defaultdict(list)
68 
69         spider_loader = self.crawler_process.spider_loader
70 
71         for spidername in args or spider_loader.list():
72             spidercls = spider_loader.load(spidername)
73             spidercls.start_requests = lambda s: conman.from_spider(s, result)
74 
75             tested_methods = conman.tested_methods_from_spidercls(spidercls)
76             if opts.list:
77                 for method in tested_methods:
78                     contract_reqs[spidercls.name].append(method)
79             elif tested_methods:
80                 self.crawler_process.crawl(spidercls)
81 
82         # start checks
83         if opts.list:
84             for spider, methods in sorted(contract_reqs.items()):
85                 if not methods and not opts.verbose:
86                     continue
87                 print(spider)
88                 for method in sorted(methods):
89                     print('  * %s' % method)
90         else:
91             start = time.time()
92             self.crawler_process.start()
93             stop = time.time()
94 
95             result.printErrors()
96             result.printSummary(start, stop)
97             self.exitcode = int(not result.wasSuccessful())
98 
99
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -6,7 +6,7 @@
 
 from scrapy.commands import ScrapyCommand
 from scrapy.contracts import ContractsManager
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, set_environ
 from scrapy.utils.conf import build_component_list
 
 
@@ -68,16 +68,17 @@
 
         spider_loader = self.crawler_process.spider_loader
 
-        for spidername in args or spider_loader.list():
-            spidercls = spider_loader.load(spidername)
-            spidercls.start_requests = lambda s: conman.from_spider(s, result)
-
-            tested_methods = conman.tested_methods_from_spidercls(spidercls)
-            if opts.list:
-                for method in tested_methods:
-                    contract_reqs[spidercls.name].append(method)
-            elif tested_methods:
-                self.crawler_process.crawl(spidercls)
+        with set_environ(SCRAPY_CHECK='true'):
+            for spidername in args or spider_loader.list():
+                spidercls = spider_loader.load(spidername)
+                spidercls.start_requests = lambda s: conman.from_spider(s, result)
+
+                tested_methods = conman.tested_methods_from_spidercls(spidercls)
+                if opts.list:
+                    for method in tested_methods:
+                        contract_reqs[spidercls.name].append(method)
+                elif tested_methods:
+                    self.crawler_process.crawl(spidercls)
 
         # start checks
         if opts.list:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -1,6 +1,8 @@
 """Helper functions which don't fit anywhere else"""
+import os
 import re
 import hashlib
+from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
 
@@ -142,3 +144,21 @@
         return objcls.from_settings(settings, *args, **kwargs)
     else:
         return objcls(*args, **kwargs)
+
+
+@contextmanager
+def set_environ(**kwargs):
+    """Temporarily set environment variables inside the context manager and
+    fully restore previous environment afterwards
+    """
+
+    original_env = {k: os.environ.get(k) for k in kwargs}
+    os.environ.update(kwargs)
+    try:
+        yield
+    finally:
+        for k, v in original_env.items():
+            if v is None:
+                del os.environ[k]
+            else:
+                os.environ[k] = v

{"golden_diff": "diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py\n--- a/scrapy/commands/check.py\n+++ b/scrapy/commands/check.py\n@@ -6,7 +6,7 @@\n \n from scrapy.commands import ScrapyCommand\n from scrapy.contracts import ContractsManager\n-from scrapy.utils.misc import load_object\n+from scrapy.utils.misc import load_object, set_environ\n from scrapy.utils.conf import build_component_list\n \n \n@@ -68,16 +68,17 @@\n \n         spider_loader = self.crawler_process.spider_loader\n \n-        for spidername in args or spider_loader.list():\n-            spidercls = spider_loader.load(spidername)\n-            spidercls.start_requests = lambda s: conman.from_spider(s, result)\n-\n-            tested_methods = conman.tested_methods_from_spidercls(spidercls)\n-            if opts.list:\n-                for method in tested_methods:\n-                    contract_reqs[spidercls.name].append(method)\n-            elif tested_methods:\n-                self.crawler_process.crawl(spidercls)\n+        with set_environ(SCRAPY_CHECK='true'):\n+            for spidername in args or spider_loader.list():\n+                spidercls = spider_loader.load(spidername)\n+                spidercls.start_requests = lambda s: conman.from_spider(s, result)\n+\n+                tested_methods = conman.tested_methods_from_spidercls(spidercls)\n+                if opts.list:\n+                    for method in tested_methods:\n+                        contract_reqs[spidercls.name].append(method)\n+                elif tested_methods:\n+                    self.crawler_process.crawl(spidercls)\n \n         # start checks\n         if opts.list:\ndiff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py\n--- a/scrapy/utils/misc.py\n+++ b/scrapy/utils/misc.py\n@@ -1,6 +1,8 @@\n \"\"\"Helper functions which don't fit anywhere else\"\"\"\n+import os\n import re\n import hashlib\n+from contextlib import contextmanager\n from importlib import import_module\n from pkgutil import iter_modules\n \n@@ -142,3 +144,21 @@\n         return objcls.from_settings(settings, *args, **kwargs)\n     else:\n         return objcls(*args, **kwargs)\n+\n+\n+@contextmanager\n+def set_environ(**kwargs):\n+    \"\"\"Temporarily set environment variables inside the context manager and\n+    fully restore previous environment afterwards\n+    \"\"\"\n+\n+    original_env = {k: os.environ.get(k) for k in kwargs}\n+    os.environ.update(kwargs)\n+    try:\n+        yield\n+    finally:\n+        for k, v in original_env.items():\n+            if v is None:\n+                del os.environ[k]\n+            else:\n+                os.environ[k] = v\n", "issue": "Set environment variable when running scrapy check\nSometimes it is nice to be able to enable/disable functionality, e.g. calculating things in settings.py when just checking spider contracts instead of running a crawl. I therefor propose setting an environment variable like `SCRAPY_CHECK` when using the check command.\n", "before_files": [{"content": "\"\"\"Helper functions which don't fit anywhere else\"\"\"\nimport re\nimport hashlib\nfrom importlib import import_module\nfrom pkgutil import iter_modules\n\nimport six\nfrom w3lib.html import replace_entities\n\nfrom scrapy.utils.python import flatten, to_unicode\nfrom scrapy.item import BaseItem\n\n\n_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes\n\n\ndef arg_to_iter(arg):\n    \"\"\"Convert an argument to an iterable. The argument can be a None, single\n    value, or an iterable.\n\n    Exception: if arg is a dict, [arg] will be returned\n    \"\"\"\n    if arg is None:\n        return []\n    elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):\n        return arg\n    else:\n        return [arg]\n\n\ndef load_object(path):\n    \"\"\"Load an object given its absolute object path, and return it.\n\n    object can be a class, function, variable or an instance.\n    path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'\n    \"\"\"\n\n    try:\n        dot = path.rindex('.')\n    except ValueError:\n        raise ValueError(\"Error loading object '%s': not a full path\" % path)\n\n    module, name = path[:dot], path[dot+1:]\n    mod = import_module(module)\n\n    try:\n        obj = getattr(mod, name)\n    except AttributeError:\n        raise NameError(\"Module '%s' doesn't define any object named '%s'\" % (module, name))\n\n    return obj\n\n\ndef walk_modules(path):\n    \"\"\"Loads a module and all its submodules from the given module path and\n    returns them. If *any* module throws an exception while importing, that\n    exception is thrown back.\n\n    For example: walk_modules('scrapy.utils')\n    \"\"\"\n\n    mods = []\n    mod = import_module(path)\n    mods.append(mod)\n    if hasattr(mod, '__path__'):\n        for _, subpath, ispkg in iter_modules(mod.__path__):\n            fullpath = path + '.' + subpath\n            if ispkg:\n                mods += walk_modules(fullpath)\n            else:\n                submod = import_module(fullpath)\n                mods.append(submod)\n    return mods\n\n\ndef extract_regex(regex, text, encoding='utf-8'):\n    \"\"\"Extract a list of unicode strings from the given text/encoding using the following policies:\n\n    * if the regex contains a named group called \"extract\" that will be returned\n    * if the regex contains multiple numbered groups, all those will be returned (flattened)\n    * if the regex doesn't contain any group the entire regex matching is returned\n    \"\"\"\n\n    if isinstance(regex, six.string_types):\n        regex = re.compile(regex, re.UNICODE)\n\n    try:\n        strings = [regex.search(text).group('extract')]   # named group\n    except Exception:\n        strings = regex.findall(text)    # full regex or numbered groups\n    strings = flatten(strings)\n\n    if isinstance(text, six.text_type):\n        return [replace_entities(s, keep=['lt', 'amp']) for s in strings]\n    else:\n        return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])\n                for s in strings]\n\n\ndef md5sum(file):\n    \"\"\"Calculate the md5 checksum of a file-like object without reading its\n    whole content in memory.\n\n    >>> from io import BytesIO\n    >>> md5sum(BytesIO(b'file content to hash'))\n    '784406af91dd5a54fbb9c84c2236595a'\n    \"\"\"\n    m = hashlib.md5()\n    while True:\n        d = file.read(8096)\n        if not d:\n            break\n        m.update(d)\n    return m.hexdigest()\n\n\ndef rel_has_nofollow(rel):\n    \"\"\"Return True if link rel attribute has nofollow type\"\"\"\n    return rel is not None and 'nofollow' in rel.split()\n\n\ndef create_instance(objcls, settings, crawler, *args, **kwargs):\n    \"\"\"Construct a class instance using its ``from_crawler`` or\n    ``from_settings`` constructors, if available.\n\n    At least one of ``settings`` and ``crawler`` needs to be different from\n    ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.\n    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be\n    tried.\n\n    ``*args`` and ``**kwargs`` are forwarded to the constructors.\n\n    Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.\n    \"\"\"\n    if settings is None:\n        if crawler is None:\n            raise ValueError(\"Specifiy at least one of settings and crawler.\")\n        settings = crawler.settings\n    if crawler and hasattr(objcls, 'from_crawler'):\n        return objcls.from_crawler(crawler, *args, **kwargs)\n    elif hasattr(objcls, 'from_settings'):\n        return objcls.from_settings(settings, *args, **kwargs)\n    else:\n        return objcls(*args, **kwargs)\n", "path": "scrapy/utils/misc.py"}, {"content": "from __future__ import print_function\nimport time\nimport sys\nfrom collections import defaultdict\nfrom unittest import TextTestRunner, TextTestResult as _TextTestResult\n\nfrom scrapy.commands import ScrapyCommand\nfrom scrapy.contracts import ContractsManager\nfrom scrapy.utils.misc import load_object\nfrom scrapy.utils.conf import build_component_list\n\n\nclass TextTestResult(_TextTestResult):\n    def printSummary(self, start, stop):\n        write = self.stream.write\n        writeln = self.stream.writeln\n\n        run = self.testsRun\n        plural = \"s\" if run != 1 else \"\"\n\n        writeln(self.separator2)\n        writeln(\"Ran %d contract%s in %.3fs\" % (run, plural, stop - start))\n        writeln()\n\n        infos = []\n        if not self.wasSuccessful():\n            write(\"FAILED\")\n            failed, errored = map(len, (self.failures, self.errors))\n            if failed:\n                infos.append(\"failures=%d\" % failed)\n            if errored:\n                infos.append(\"errors=%d\" % errored)\n        else:\n            write(\"OK\")\n\n        if infos:\n            writeln(\" (%s)\" % (\", \".join(infos),))\n        else:\n            write(\"\\n\")\n\n\nclass Command(ScrapyCommand):\n    requires_project = True\n    default_settings = {'LOG_ENABLED': False}\n\n    def syntax(self):\n        return \"[options] <spider>\"\n\n    def short_desc(self):\n        return \"Check spider contracts\"\n\n    def add_options(self, parser):\n        ScrapyCommand.add_options(self, parser)\n        parser.add_option(\"-l\", \"--list\", dest=\"list\", action=\"store_true\",\n                          help=\"only list contracts, without checking them\")\n        parser.add_option(\"-v\", \"--verbose\", dest=\"verbose\", default=False, action='store_true',\n                          help=\"print contract tests for all spiders\")\n\n    def run(self, args, opts):\n        # load contracts\n        contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS'))\n        conman = ContractsManager(load_object(c) for c in contracts)\n        runner = TextTestRunner(verbosity=2 if opts.verbose else 1)\n        result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)\n\n        # contract requests\n        contract_reqs = defaultdict(list)\n\n        spider_loader = self.crawler_process.spider_loader\n\n        for spidername in args or spider_loader.list():\n            spidercls = spider_loader.load(spidername)\n            spidercls.start_requests = lambda s: conman.from_spider(s, result)\n\n            tested_methods = conman.tested_methods_from_spidercls(spidercls)\n            if opts.list:\n                for method in tested_methods:\n                    contract_reqs[spidercls.name].append(method)\n            elif tested_methods:\n                self.crawler_process.crawl(spidercls)\n\n        # start checks\n        if opts.list:\n            for spider, methods in sorted(contract_reqs.items()):\n                if not methods and not opts.verbose:\n                    continue\n                print(spider)\n                for method in sorted(methods):\n                    print('  * %s' % method)\n        else:\n            start = time.time()\n            self.crawler_process.start()\n            stop = time.time()\n\n            result.printErrors()\n            result.printSummary(start, stop)\n            self.exitcode = int(not result.wasSuccessful())\n\n", "path": "scrapy/commands/check.py"}], "after_files": [{"content": "\"\"\"Helper functions which don't fit anywhere else\"\"\"\nimport os\nimport re\nimport hashlib\nfrom contextlib import contextmanager\nfrom importlib import import_module\nfrom pkgutil import iter_modules\n\nimport six\nfrom w3lib.html import replace_entities\n\nfrom scrapy.utils.python import flatten, to_unicode\nfrom scrapy.item import BaseItem\n\n\n_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes\n\n\ndef arg_to_iter(arg):\n    \"\"\"Convert an argument to an iterable. The argument can be a None, single\n    value, or an iterable.\n\n    Exception: if arg is a dict, [arg] will be returned\n    \"\"\"\n    if arg is None:\n        return []\n    elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):\n        return arg\n    else:\n        return [arg]\n\n\ndef load_object(path):\n    \"\"\"Load an object given its absolute object path, and return it.\n\n    object can be a class, function, variable or an instance.\n    path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'\n    \"\"\"\n\n    try:\n        dot = path.rindex('.')\n    except ValueError:\n        raise ValueError(\"Error loading object '%s': not a full path\" % path)\n\n    module, name = path[:dot], path[dot+1:]\n    mod = import_module(module)\n\n    try:\n        obj = getattr(mod, name)\n    except AttributeError:\n        raise NameError(\"Module '%s' doesn't define any object named '%s'\" % (module, name))\n\n    return obj\n\n\ndef walk_modules(path):\n    \"\"\"Loads a module and all its submodules from the given module path and\n    returns them. If *any* module throws an exception while importing, that\n    exception is thrown back.\n\n    For example: walk_modules('scrapy.utils')\n    \"\"\"\n\n    mods = []\n    mod = import_module(path)\n    mods.append(mod)\n    if hasattr(mod, '__path__'):\n        for _, subpath, ispkg in iter_modules(mod.__path__):\n            fullpath = path + '.' + subpath\n            if ispkg:\n                mods += walk_modules(fullpath)\n            else:\n                submod = import_module(fullpath)\n                mods.append(submod)\n    return mods\n\n\ndef extract_regex(regex, text, encoding='utf-8'):\n    \"\"\"Extract a list of unicode strings from the given text/encoding using the following policies:\n\n    * if the regex contains a named group called \"extract\" that will be returned\n    * if the regex contains multiple numbered groups, all those will be returned (flattened)\n    * if the regex doesn't contain any group the entire regex matching is returned\n    \"\"\"\n\n    if isinstance(regex, six.string_types):\n        regex = re.compile(regex, re.UNICODE)\n\n    try:\n        strings = [regex.search(text).group('extract')]   # named group\n    except Exception:\n        strings = regex.findall(text)    # full regex or numbered groups\n    strings = flatten(strings)\n\n    if isinstance(text, six.text_type):\n        return [replace_entities(s, keep=['lt', 'amp']) for s in strings]\n    else:\n        return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])\n                for s in strings]\n\n\ndef md5sum(file):\n    \"\"\"Calculate the md5 checksum of a file-like object without reading its\n    whole content in memory.\n\n    >>> from io import BytesIO\n    >>> md5sum(BytesIO(b'file content to hash'))\n    '784406af91dd5a54fbb9c84c2236595a'\n    \"\"\"\n    m = hashlib.md5()\n    while True:\n        d = file.read(8096)\n        if not d:\n            break\n        m.update(d)\n    return m.hexdigest()\n\n\ndef rel_has_nofollow(rel):\n    \"\"\"Return True if link rel attribute has nofollow type\"\"\"\n    return rel is not None and 'nofollow' in rel.split()\n\n\ndef create_instance(objcls, settings, crawler, *args, **kwargs):\n    \"\"\"Construct a class instance using its ``from_crawler`` or\n    ``from_settings`` constructors, if available.\n\n    At least one of ``settings`` and ``crawler`` needs to be different from\n    ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.\n    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be\n    tried.\n\n    ``*args`` and ``**kwargs`` are forwarded to the constructors.\n\n    Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.\n    \"\"\"\n    if settings is None:\n        if crawler is None:\n            raise ValueError(\"Specifiy at least one of settings and crawler.\")\n        settings = crawler.settings\n    if crawler and hasattr(objcls, 'from_crawler'):\n        return objcls.from_crawler(crawler, *args, **kwargs)\n    elif hasattr(objcls, 'from_settings'):\n        return objcls.from_settings(settings, *args, **kwargs)\n    else:\n        return objcls(*args, **kwargs)\n\n\n@contextmanager\ndef set_environ(**kwargs):\n    \"\"\"Temporarily set environment variables inside the context manager and\n    fully restore previous environment afterwards\n    \"\"\"\n\n    original_env = {k: os.environ.get(k) for k in kwargs}\n    os.environ.update(kwargs)\n    try:\n        yield\n    finally:\n        for k, v in original_env.items():\n            if v is None:\n                del os.environ[k]\n            else:\n                os.environ[k] = v\n", "path": "scrapy/utils/misc.py"}, {"content": "from __future__ import print_function\nimport time\nimport sys\nfrom collections import defaultdict\nfrom unittest import TextTestRunner, TextTestResult as _TextTestResult\n\nfrom scrapy.commands import ScrapyCommand\nfrom scrapy.contracts import ContractsManager\nfrom scrapy.utils.misc import load_object, set_environ\nfrom scrapy.utils.conf import build_component_list\n\n\nclass TextTestResult(_TextTestResult):\n    def printSummary(self, start, stop):\n        write = self.stream.write\n        writeln = self.stream.writeln\n\n        run = self.testsRun\n        plural = \"s\" if run != 1 else \"\"\n\n        writeln(self.separator2)\n        writeln(\"Ran %d contract%s in %.3fs\" % (run, plural, stop - start))\n        writeln()\n\n        infos = []\n        if not self.wasSuccessful():\n            write(\"FAILED\")\n            failed, errored = map(len, (self.failures, self.errors))\n            if failed:\n                infos.append(\"failures=%d\" % failed)\n            if errored:\n                infos.append(\"errors=%d\" % errored)\n        else:\n            write(\"OK\")\n\n        if infos:\n            writeln(\" (%s)\" % (\", \".join(infos),))\n        else:\n            write(\"\\n\")\n\n\nclass Command(ScrapyCommand):\n    requires_project = True\n    default_settings = {'LOG_ENABLED': False}\n\n    def syntax(self):\n        return \"[options] <spider>\"\n\n    def short_desc(self):\n        return \"Check spider contracts\"\n\n    def add_options(self, parser):\n        ScrapyCommand.add_options(self, parser)\n        parser.add_option(\"-l\", \"--list\", dest=\"list\", action=\"store_true\",\n                          help=\"only list contracts, without checking them\")\n        parser.add_option(\"-v\", \"--verbose\", dest=\"verbose\", default=False, action='store_true',\n                          help=\"print contract tests for all spiders\")\n\n    def run(self, args, opts):\n        # load contracts\n        contracts = build_component_list(self.settings.getwithbase('SPIDER_CONTRACTS'))\n        conman = ContractsManager(load_object(c) for c in contracts)\n        runner = TextTestRunner(verbosity=2 if opts.verbose else 1)\n        result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity)\n\n        # contract requests\n        contract_reqs = defaultdict(list)\n\n        spider_loader = self.crawler_process.spider_loader\n\n        with set_environ(SCRAPY_CHECK='true'):\n            for spidername in args or spider_loader.list():\n                spidercls = spider_loader.load(spidername)\n                spidercls.start_requests = lambda s: conman.from_spider(s, result)\n\n                tested_methods = conman.tested_methods_from_spidercls(spidercls)\n                if opts.list:\n                    for method in tested_methods:\n                        contract_reqs[spidercls.name].append(method)\n                elif tested_methods:\n                    self.crawler_process.crawl(spidercls)\n\n        # start checks\n        if opts.list:\n            for spider, methods in sorted(contract_reqs.items()):\n                if not methods and not opts.verbose:\n                    continue\n                print(spider)\n                for method in sorted(methods):\n                    print('  * %s' % method)\n        else:\n            start = time.time()\n            self.crawler_process.start()\n            stop = time.time()\n\n            result.printErrors()\n            result.printSummary(start, stop)\n            self.exitcode = int(not result.wasSuccessful())\n\n", "path": "scrapy/commands/check.py"}]}