We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Smarter generator check for combined yield/return statements: ignore nested functions
## Summary
Currently, if spider methods are generators that yield results and contain nested function, then the following warning is issued:

```
[py.warnings] WARNING: /Library/Python/3.7/site-packages/scrapy/core/scraper.py:148: UserWarning: The "MySpider.parse" method is a generator and includes a "return" statement with a value different than None. This could lead to unexpected behaviour. Please see https://docs.python.org/3/reference/simple_stmts.html#the-return-statement for details about the semantics of the "return" statement within generators
  warn_on_generator_with_return_value(spider, callback)
```

The example of a simple spider that results in the warning:
```
import scrapy

class MySpider(scrapy.Spider):
    name = "MySpider"
    start_urls = ["https://scrapy.org"]
    
    def parse(self, response):
        
        def is_external(url):
            href = url.css('::attr(href)').get()
            return href.startswith('http') and 'scrapy.org' not in href
        
        links = [link for link in response.css('a') if is_external(link)]
        for link in links:
            yield {'link': link.css('::attr(href)').get(), 'text': link.css('::text').get()}
```

I know it's a bit artificial example as the nested function can be moved, but there is nothing wrong with nested function conceptually.

## Motivation

I have a midsize spider function that includes some nested helper functions that I'd like to keep close to where they are called.

## Describe alternatives you've considered

Moving nested function out of the generator is an easy fix, but it constrains expressivity of the code.

## Additional context

Related function: is_generator_with_return_value


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/utils/misc.py`
Content:
```
1 """Helper functions which don't fit anywhere else"""
2 import ast
3 import inspect
4 import os
5 import re
6 import hashlib
7 import warnings
8 from contextlib import contextmanager
9 from importlib import import_module
10 from pkgutil import iter_modules
11 from textwrap import dedent
12 
13 from w3lib.html import replace_entities
14 
15 from scrapy.utils.datatypes import LocalWeakReferencedCache
16 from scrapy.utils.python import flatten, to_unicode
17 from scrapy.item import _BaseItem
18 from scrapy.utils.deprecate import ScrapyDeprecationWarning
19 
20 
21 _ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
22 
23 
24 def arg_to_iter(arg):
25     """Convert an argument to an iterable. The argument can be a None, single
26     value, or an iterable.
27 
28     Exception: if arg is a dict, [arg] will be returned
29     """
30     if arg is None:
31         return []
32     elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):
33         return arg
34     else:
35         return [arg]
36 
37 
38 def load_object(path):
39     """Load an object given its absolute object path, and return it.
40 
41     object can be the import path of a class, function, variable or an
42     instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'
43     """
44 
45     try:
46         dot = path.rindex('.')
47     except ValueError:
48         raise ValueError("Error loading object '%s': not a full path" % path)
49 
50     module, name = path[:dot], path[dot + 1:]
51     mod = import_module(module)
52 
53     try:
54         obj = getattr(mod, name)
55     except AttributeError:
56         raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name))
57 
58     return obj
59 
60 
61 def walk_modules(path):
62     """Loads a module and all its submodules from the given module path and
63     returns them. If *any* module throws an exception while importing, that
64     exception is thrown back.
65 
66     For example: walk_modules('scrapy.utils')
67     """
68 
69     mods = []
70     mod = import_module(path)
71     mods.append(mod)
72     if hasattr(mod, '__path__'):
73         for _, subpath, ispkg in iter_modules(mod.__path__):
74             fullpath = path + '.' + subpath
75             if ispkg:
76                 mods += walk_modules(fullpath)
77             else:
78                 submod = import_module(fullpath)
79                 mods.append(submod)
80     return mods
81 
82 
83 def extract_regex(regex, text, encoding='utf-8'):
84     """Extract a list of unicode strings from the given text/encoding using the following policies:
85 
86     * if the regex contains a named group called "extract" that will be returned
87     * if the regex contains multiple numbered groups, all those will be returned (flattened)
88     * if the regex doesn't contain any group the entire regex matching is returned
89     """
90     warnings.warn(
91         "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
92         ScrapyDeprecationWarning,
93         stacklevel=2
94     )
95 
96     if isinstance(regex, str):
97         regex = re.compile(regex, re.UNICODE)
98 
99     try:
100         strings = [regex.search(text).group('extract')]   # named group
101     except Exception:
102         strings = regex.findall(text)    # full regex or numbered groups
103     strings = flatten(strings)
104 
105     if isinstance(text, str):
106         return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
107     else:
108         return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
109                 for s in strings]
110 
111 
112 def md5sum(file):
113     """Calculate the md5 checksum of a file-like object without reading its
114     whole content in memory.
115 
116     >>> from io import BytesIO
117     >>> md5sum(BytesIO(b'file content to hash'))
118     '784406af91dd5a54fbb9c84c2236595a'
119     """
120     m = hashlib.md5()
121     while True:
122         d = file.read(8096)
123         if not d:
124             break
125         m.update(d)
126     return m.hexdigest()
127 
128 
129 def rel_has_nofollow(rel):
130     """Return True if link rel attribute has nofollow type"""
131     return rel is not None and 'nofollow' in rel.split()
132 
133 
134 def create_instance(objcls, settings, crawler, *args, **kwargs):
135     """Construct a class instance using its ``from_crawler`` or
136     ``from_settings`` constructors, if available.
137 
138     At least one of ``settings`` and ``crawler`` needs to be different from
139     ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
140     If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
141     tried.
142 
143     ``*args`` and ``**kwargs`` are forwarded to the constructors.
144 
145     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
146 
147     .. versionchanged:: 2.2
148        Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
149        extension has not been implemented correctly).
150     """
151     if settings is None:
152         if crawler is None:
153             raise ValueError("Specify at least one of settings and crawler.")
154         settings = crawler.settings
155     if crawler and hasattr(objcls, 'from_crawler'):
156         instance = objcls.from_crawler(crawler, *args, **kwargs)
157         method_name = 'from_crawler'
158     elif hasattr(objcls, 'from_settings'):
159         instance = objcls.from_settings(settings, *args, **kwargs)
160         method_name = 'from_settings'
161     else:
162         instance = objcls(*args, **kwargs)
163         method_name = '__new__'
164     if instance is None:
165         raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name))
166     return instance
167 
168 
169 @contextmanager
170 def set_environ(**kwargs):
171     """Temporarily set environment variables inside the context manager and
172     fully restore previous environment afterwards
173     """
174 
175     original_env = {k: os.environ.get(k) for k in kwargs}
176     os.environ.update(kwargs)
177     try:
178         yield
179     finally:
180         for k, v in original_env.items():
181             if v is None:
182                 del os.environ[k]
183             else:
184                 os.environ[k] = v
185 
186 
187 _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
188 
189 
190 def is_generator_with_return_value(callable):
191     """
192     Returns True if a callable is a generator function which includes a
193     'return' statement with a value different than None, False otherwise
194     """
195     if callable in _generator_callbacks_cache:
196         return _generator_callbacks_cache[callable]
197 
198     def returns_none(return_node):
199         value = return_node.value
200         return value is None or isinstance(value, ast.NameConstant) and value.value is None
201 
202     if inspect.isgeneratorfunction(callable):
203         tree = ast.parse(dedent(inspect.getsource(callable)))
204         for node in ast.walk(tree):
205             if isinstance(node, ast.Return) and not returns_none(node):
206                 _generator_callbacks_cache[callable] = True
207                 return _generator_callbacks_cache[callable]
208 
209     _generator_callbacks_cache[callable] = False
210     return _generator_callbacks_cache[callable]
211 
212 
213 def warn_on_generator_with_return_value(spider, callable):
214     """
215     Logs a warning if a callable is a generator function and includes
216     a 'return' statement with a value different than None
217     """
218     if is_generator_with_return_value(callable):
219         warnings.warn(
220             'The "{}.{}" method is a generator and includes a "return" statement with a '
221             'value different than None. This could lead to unexpected behaviour. Please see '
222             'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
223             'for details about the semantics of the "return" statement within generators'
224             .format(spider.__class__.__name__, callable.__name__), stacklevel=2,
225         )
226
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -5,6 +5,7 @@
 import re
 import hashlib
 import warnings
+from collections import deque
 from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
@@ -184,6 +185,22 @@
                 os.environ[k] = v
 
 
+def walk_callable(node):
+    """Similar to ``ast.walk``, but walks only function body and skips nested
+    functions defined within the node.
+    """
+    todo = deque([node])
+    walked_func_def = False
+    while todo:
+        node = todo.popleft()
+        if isinstance(node, ast.FunctionDef):
+            if walked_func_def:
+                continue
+            walked_func_def = True
+        todo.extend(ast.iter_child_nodes(node))
+        yield node
+
+
 _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
 
 
@@ -201,7 +218,7 @@
 
     if inspect.isgeneratorfunction(callable):
         tree = ast.parse(dedent(inspect.getsource(callable)))
-        for node in ast.walk(tree):
+        for node in walk_callable(tree):
             if isinstance(node, ast.Return) and not returns_none(node):
                 _generator_callbacks_cache[callable] = True
                 return _generator_callbacks_cache[callable]

{"golden_diff": "diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py\n--- a/scrapy/utils/misc.py\n+++ b/scrapy/utils/misc.py\n@@ -5,6 +5,7 @@\n import re\n import hashlib\n import warnings\n+from collections import deque\n from contextlib import contextmanager\n from importlib import import_module\n from pkgutil import iter_modules\n@@ -184,6 +185,22 @@\n                 os.environ[k] = v\n \n \n+def walk_callable(node):\n+    \"\"\"Similar to ``ast.walk``, but walks only function body and skips nested\n+    functions defined within the node.\n+    \"\"\"\n+    todo = deque([node])\n+    walked_func_def = False\n+    while todo:\n+        node = todo.popleft()\n+        if isinstance(node, ast.FunctionDef):\n+            if walked_func_def:\n+                continue\n+            walked_func_def = True\n+        todo.extend(ast.iter_child_nodes(node))\n+        yield node\n+\n+\n _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)\n \n \n@@ -201,7 +218,7 @@\n \n     if inspect.isgeneratorfunction(callable):\n         tree = ast.parse(dedent(inspect.getsource(callable)))\n-        for node in ast.walk(tree):\n+        for node in walk_callable(tree):\n             if isinstance(node, ast.Return) and not returns_none(node):\n                 _generator_callbacks_cache[callable] = True\n                 return _generator_callbacks_cache[callable]\n", "issue": "Smarter generator check for combined yield/return statements: ignore nested functions\n## Summary\r\nCurrently, if spider methods are generators that yield results and contain nested function, then the following warning is issued:\r\n\r\n```\r\n[py.warnings] WARNING: /Library/Python/3.7/site-packages/scrapy/core/scraper.py:148: UserWarning: The \"MySpider.parse\" method is a generator and includes a \"return\" statement with a value different than None. This could lead to unexpected behaviour. Please see https://docs.python.org/3/reference/simple_stmts.html#the-return-statement for details about the semantics of the \"return\" statement within generators\r\n  warn_on_generator_with_return_value(spider, callback)\r\n```\r\n\r\nThe example of a simple spider that results in the warning:\r\n```\r\nimport scrapy\r\n\r\nclass MySpider(scrapy.Spider):\r\n    name = \"MySpider\"\r\n    start_urls = [\"https://scrapy.org\"]\r\n    \r\n    def parse(self, response):\r\n        \r\n        def is_external(url):\r\n            href = url.css('::attr(href)').get()\r\n            return href.startswith('http') and 'scrapy.org' not in href\r\n        \r\n        links = [link for link in response.css('a') if is_external(link)]\r\n        for link in links:\r\n            yield {'link': link.css('::attr(href)').get(), 'text': link.css('::text').get()}\r\n```\r\n\r\nI know it's a bit artificial example as the nested function can be moved, but there is nothing wrong with nested function conceptually.\r\n\r\n## Motivation\r\n\r\nI have a midsize spider function that includes some nested helper functions that I'd like to keep close to where they are called.\r\n\r\n## Describe alternatives you've considered\r\n\r\nMoving nested function out of the generator is an easy fix, but it constrains expressivity of the code.\r\n\r\n## Additional context\r\n\r\nRelated function: is_generator_with_return_value\r\n\n", "before_files": [{"content": "\"\"\"Helper functions which don't fit anywhere else\"\"\"\nimport ast\nimport inspect\nimport os\nimport re\nimport hashlib\nimport warnings\nfrom contextlib import contextmanager\nfrom importlib import import_module\nfrom pkgutil import iter_modules\nfrom textwrap import dedent\n\nfrom w3lib.html import replace_entities\n\nfrom scrapy.utils.datatypes import LocalWeakReferencedCache\nfrom scrapy.utils.python import flatten, to_unicode\nfrom scrapy.item import _BaseItem\nfrom scrapy.utils.deprecate import ScrapyDeprecationWarning\n\n\n_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes\n\n\ndef arg_to_iter(arg):\n    \"\"\"Convert an argument to an iterable. The argument can be a None, single\n    value, or an iterable.\n\n    Exception: if arg is a dict, [arg] will be returned\n    \"\"\"\n    if arg is None:\n        return []\n    elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):\n        return arg\n    else:\n        return [arg]\n\n\ndef load_object(path):\n    \"\"\"Load an object given its absolute object path, and return it.\n\n    object can be the import path of a class, function, variable or an\n    instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'\n    \"\"\"\n\n    try:\n        dot = path.rindex('.')\n    except ValueError:\n        raise ValueError(\"Error loading object '%s': not a full path\" % path)\n\n    module, name = path[:dot], path[dot + 1:]\n    mod = import_module(module)\n\n    try:\n        obj = getattr(mod, name)\n    except AttributeError:\n        raise NameError(\"Module '%s' doesn't define any object named '%s'\" % (module, name))\n\n    return obj\n\n\ndef walk_modules(path):\n    \"\"\"Loads a module and all its submodules from the given module path and\n    returns them. If *any* module throws an exception while importing, that\n    exception is thrown back.\n\n    For example: walk_modules('scrapy.utils')\n    \"\"\"\n\n    mods = []\n    mod = import_module(path)\n    mods.append(mod)\n    if hasattr(mod, '__path__'):\n        for _, subpath, ispkg in iter_modules(mod.__path__):\n            fullpath = path + '.' + subpath\n            if ispkg:\n                mods += walk_modules(fullpath)\n            else:\n                submod = import_module(fullpath)\n                mods.append(submod)\n    return mods\n\n\ndef extract_regex(regex, text, encoding='utf-8'):\n    \"\"\"Extract a list of unicode strings from the given text/encoding using the following policies:\n\n    * if the regex contains a named group called \"extract\" that will be returned\n    * if the regex contains multiple numbered groups, all those will be returned (flattened)\n    * if the regex doesn't contain any group the entire regex matching is returned\n    \"\"\"\n    warnings.warn(\n        \"scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.\",\n        ScrapyDeprecationWarning,\n        stacklevel=2\n    )\n\n    if isinstance(regex, str):\n        regex = re.compile(regex, re.UNICODE)\n\n    try:\n        strings = [regex.search(text).group('extract')]   # named group\n    except Exception:\n        strings = regex.findall(text)    # full regex or numbered groups\n    strings = flatten(strings)\n\n    if isinstance(text, str):\n        return [replace_entities(s, keep=['lt', 'amp']) for s in strings]\n    else:\n        return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])\n                for s in strings]\n\n\ndef md5sum(file):\n    \"\"\"Calculate the md5 checksum of a file-like object without reading its\n    whole content in memory.\n\n    >>> from io import BytesIO\n    >>> md5sum(BytesIO(b'file content to hash'))\n    '784406af91dd5a54fbb9c84c2236595a'\n    \"\"\"\n    m = hashlib.md5()\n    while True:\n        d = file.read(8096)\n        if not d:\n            break\n        m.update(d)\n    return m.hexdigest()\n\n\ndef rel_has_nofollow(rel):\n    \"\"\"Return True if link rel attribute has nofollow type\"\"\"\n    return rel is not None and 'nofollow' in rel.split()\n\n\ndef create_instance(objcls, settings, crawler, *args, **kwargs):\n    \"\"\"Construct a class instance using its ``from_crawler`` or\n    ``from_settings`` constructors, if available.\n\n    At least one of ``settings`` and ``crawler`` needs to be different from\n    ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.\n    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be\n    tried.\n\n    ``*args`` and ``**kwargs`` are forwarded to the constructors.\n\n    Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.\n\n    .. versionchanged:: 2.2\n       Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an\n       extension has not been implemented correctly).\n    \"\"\"\n    if settings is None:\n        if crawler is None:\n            raise ValueError(\"Specify at least one of settings and crawler.\")\n        settings = crawler.settings\n    if crawler and hasattr(objcls, 'from_crawler'):\n        instance = objcls.from_crawler(crawler, *args, **kwargs)\n        method_name = 'from_crawler'\n    elif hasattr(objcls, 'from_settings'):\n        instance = objcls.from_settings(settings, *args, **kwargs)\n        method_name = 'from_settings'\n    else:\n        instance = objcls(*args, **kwargs)\n        method_name = '__new__'\n    if instance is None:\n        raise TypeError(\"%s.%s returned None\" % (objcls.__qualname__, method_name))\n    return instance\n\n\n@contextmanager\ndef set_environ(**kwargs):\n    \"\"\"Temporarily set environment variables inside the context manager and\n    fully restore previous environment afterwards\n    \"\"\"\n\n    original_env = {k: os.environ.get(k) for k in kwargs}\n    os.environ.update(kwargs)\n    try:\n        yield\n    finally:\n        for k, v in original_env.items():\n            if v is None:\n                del os.environ[k]\n            else:\n                os.environ[k] = v\n\n\n_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)\n\n\ndef is_generator_with_return_value(callable):\n    \"\"\"\n    Returns True if a callable is a generator function which includes a\n    'return' statement with a value different than None, False otherwise\n    \"\"\"\n    if callable in _generator_callbacks_cache:\n        return _generator_callbacks_cache[callable]\n\n    def returns_none(return_node):\n        value = return_node.value\n        return value is None or isinstance(value, ast.NameConstant) and value.value is None\n\n    if inspect.isgeneratorfunction(callable):\n        tree = ast.parse(dedent(inspect.getsource(callable)))\n        for node in ast.walk(tree):\n            if isinstance(node, ast.Return) and not returns_none(node):\n                _generator_callbacks_cache[callable] = True\n                return _generator_callbacks_cache[callable]\n\n    _generator_callbacks_cache[callable] = False\n    return _generator_callbacks_cache[callable]\n\n\ndef warn_on_generator_with_return_value(spider, callable):\n    \"\"\"\n    Logs a warning if a callable is a generator function and includes\n    a 'return' statement with a value different than None\n    \"\"\"\n    if is_generator_with_return_value(callable):\n        warnings.warn(\n            'The \"{}.{}\" method is a generator and includes a \"return\" statement with a '\n            'value different than None. This could lead to unexpected behaviour. Please see '\n            'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '\n            'for details about the semantics of the \"return\" statement within generators'\n            .format(spider.__class__.__name__, callable.__name__), stacklevel=2,\n        )\n", "path": "scrapy/utils/misc.py"}], "after_files": [{"content": "\"\"\"Helper functions which don't fit anywhere else\"\"\"\nimport ast\nimport inspect\nimport os\nimport re\nimport hashlib\nimport warnings\nfrom collections import deque\nfrom contextlib import contextmanager\nfrom importlib import import_module\nfrom pkgutil import iter_modules\nfrom textwrap import dedent\n\nfrom w3lib.html import replace_entities\n\nfrom scrapy.utils.datatypes import LocalWeakReferencedCache\nfrom scrapy.utils.python import flatten, to_unicode\nfrom scrapy.item import _BaseItem\nfrom scrapy.utils.deprecate import ScrapyDeprecationWarning\n\n\n_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes\n\n\ndef arg_to_iter(arg):\n    \"\"\"Convert an argument to an iterable. The argument can be a None, single\n    value, or an iterable.\n\n    Exception: if arg is a dict, [arg] will be returned\n    \"\"\"\n    if arg is None:\n        return []\n    elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):\n        return arg\n    else:\n        return [arg]\n\n\ndef load_object(path):\n    \"\"\"Load an object given its absolute object path, and return it.\n\n    object can be the import path of a class, function, variable or an\n    instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'\n    \"\"\"\n\n    try:\n        dot = path.rindex('.')\n    except ValueError:\n        raise ValueError(\"Error loading object '%s': not a full path\" % path)\n\n    module, name = path[:dot], path[dot + 1:]\n    mod = import_module(module)\n\n    try:\n        obj = getattr(mod, name)\n    except AttributeError:\n        raise NameError(\"Module '%s' doesn't define any object named '%s'\" % (module, name))\n\n    return obj\n\n\ndef walk_modules(path):\n    \"\"\"Loads a module and all its submodules from the given module path and\n    returns them. If *any* module throws an exception while importing, that\n    exception is thrown back.\n\n    For example: walk_modules('scrapy.utils')\n    \"\"\"\n\n    mods = []\n    mod = import_module(path)\n    mods.append(mod)\n    if hasattr(mod, '__path__'):\n        for _, subpath, ispkg in iter_modules(mod.__path__):\n            fullpath = path + '.' + subpath\n            if ispkg:\n                mods += walk_modules(fullpath)\n            else:\n                submod = import_module(fullpath)\n                mods.append(submod)\n    return mods\n\n\ndef extract_regex(regex, text, encoding='utf-8'):\n    \"\"\"Extract a list of unicode strings from the given text/encoding using the following policies:\n\n    * if the regex contains a named group called \"extract\" that will be returned\n    * if the regex contains multiple numbered groups, all those will be returned (flattened)\n    * if the regex doesn't contain any group the entire regex matching is returned\n    \"\"\"\n    warnings.warn(\n        \"scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.\",\n        ScrapyDeprecationWarning,\n        stacklevel=2\n    )\n\n    if isinstance(regex, str):\n        regex = re.compile(regex, re.UNICODE)\n\n    try:\n        strings = [regex.search(text).group('extract')]   # named group\n    except Exception:\n        strings = regex.findall(text)    # full regex or numbered groups\n    strings = flatten(strings)\n\n    if isinstance(text, str):\n        return [replace_entities(s, keep=['lt', 'amp']) for s in strings]\n    else:\n        return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])\n                for s in strings]\n\n\ndef md5sum(file):\n    \"\"\"Calculate the md5 checksum of a file-like object without reading its\n    whole content in memory.\n\n    >>> from io import BytesIO\n    >>> md5sum(BytesIO(b'file content to hash'))\n    '784406af91dd5a54fbb9c84c2236595a'\n    \"\"\"\n    m = hashlib.md5()\n    while True:\n        d = file.read(8096)\n        if not d:\n            break\n        m.update(d)\n    return m.hexdigest()\n\n\ndef rel_has_nofollow(rel):\n    \"\"\"Return True if link rel attribute has nofollow type\"\"\"\n    return rel is not None and 'nofollow' in rel.split()\n\n\ndef create_instance(objcls, settings, crawler, *args, **kwargs):\n    \"\"\"Construct a class instance using its ``from_crawler`` or\n    ``from_settings`` constructors, if available.\n\n    At least one of ``settings`` and ``crawler`` needs to be different from\n    ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.\n    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be\n    tried.\n\n    ``*args`` and ``**kwargs`` are forwarded to the constructors.\n\n    Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.\n\n    .. versionchanged:: 2.2\n       Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an\n       extension has not been implemented correctly).\n    \"\"\"\n    if settings is None:\n        if crawler is None:\n            raise ValueError(\"Specify at least one of settings and crawler.\")\n        settings = crawler.settings\n    if crawler and hasattr(objcls, 'from_crawler'):\n        instance = objcls.from_crawler(crawler, *args, **kwargs)\n        method_name = 'from_crawler'\n    elif hasattr(objcls, 'from_settings'):\n        instance = objcls.from_settings(settings, *args, **kwargs)\n        method_name = 'from_settings'\n    else:\n        instance = objcls(*args, **kwargs)\n        method_name = '__new__'\n    if instance is None:\n        raise TypeError(\"%s.%s returned None\" % (objcls.__qualname__, method_name))\n    return instance\n\n\n@contextmanager\ndef set_environ(**kwargs):\n    \"\"\"Temporarily set environment variables inside the context manager and\n    fully restore previous environment afterwards\n    \"\"\"\n\n    original_env = {k: os.environ.get(k) for k in kwargs}\n    os.environ.update(kwargs)\n    try:\n        yield\n    finally:\n        for k, v in original_env.items():\n            if v is None:\n                del os.environ[k]\n            else:\n                os.environ[k] = v\n\n\ndef walk_callable(node):\n    \"\"\"Similar to ``ast.walk``, but walks only function body and skips nested\n    functions defined within the node.\n    \"\"\"\n    todo = deque([node])\n    walked_func_def = False\n    while todo:\n        node = todo.popleft()\n        if isinstance(node, ast.FunctionDef):\n            if walked_func_def:\n                continue\n            walked_func_def = True\n        todo.extend(ast.iter_child_nodes(node))\n        yield node\n\n\n_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)\n\n\ndef is_generator_with_return_value(callable):\n    \"\"\"\n    Returns True if a callable is a generator function which includes a\n    'return' statement with a value different than None, False otherwise\n    \"\"\"\n    if callable in _generator_callbacks_cache:\n        return _generator_callbacks_cache[callable]\n\n    def returns_none(return_node):\n        value = return_node.value\n        return value is None or isinstance(value, ast.NameConstant) and value.value is None\n\n    if inspect.isgeneratorfunction(callable):\n        tree = ast.parse(dedent(inspect.getsource(callable)))\n        for node in walk_callable(tree):\n            if isinstance(node, ast.Return) and not returns_none(node):\n                _generator_callbacks_cache[callable] = True\n                return _generator_callbacks_cache[callable]\n\n    _generator_callbacks_cache[callable] = False\n    return _generator_callbacks_cache[callable]\n\n\ndef warn_on_generator_with_return_value(spider, callable):\n    \"\"\"\n    Logs a warning if a callable is a generator function and includes\n    a 'return' statement with a value different than None\n    \"\"\"\n    if is_generator_with_return_value(callable):\n        warnings.warn(\n            'The \"{}.{}\" method is a generator and includes a \"return\" statement with a '\n            'value different than None. This could lead to unexpected behaviour. Please see '\n            'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '\n            'for details about the semantics of the \"return\" statement within generators'\n            .format(spider.__class__.__name__, callable.__name__), stacklevel=2,\n        )\n", "path": "scrapy/utils/misc.py"}]}