We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Backward incompatibility for relocated paths in settings
Reported by @dangra

This issue manifests when mixing old paths and new ones for extensions and middlewares (this can happen for example while using a newer version of Scrapy in a project that hasn't updated to the new paths yet). Since paths aren't normalized, the same component can be loaded twice.

Take these settings for example:

``` python
# scrapy/settings/default_settings.py
EXTENSIONS_BASE = {
    'scrapy.extensions.debug.StackTraceDump': 100,  # new path
} 
```

``` python
# myproject/settings.py
EXTENSIONS = {
    'scrapy.contrib.debug.StackTraceDump': 200,  # old path
}
```

While merging both dictionaries to build the list of components, the same StackTraceDump class is going to be loaded twice since it appears in two different keys. 


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/utils/deprecate.py`
Content:
```
1 """Some helpers for deprecation messages"""
2 
3 import warnings
4 import inspect
5 from scrapy.exceptions import ScrapyDeprecationWarning
6 
7 
8 def attribute(obj, oldattr, newattr, version='0.12'):
9     cname = obj.__class__.__name__
10     warnings.warn("%s.%s attribute is deprecated and will be no longer supported "
11         "in Scrapy %s, use %s.%s attribute instead" % \
12         (cname, oldattr, version, cname, newattr), ScrapyDeprecationWarning, stacklevel=3)
13 
14 
15 def create_deprecated_class(name, new_class, clsdict=None,
16                             warn_category=ScrapyDeprecationWarning,
17                             warn_once=True,
18                             old_class_path=None,
19                             new_class_path=None,
20                             subclass_warn_message="{cls} inherits from "\
21                                     "deprecated class {old}, please inherit "\
22                                     "from {new}.",
23                             instance_warn_message="{cls} is deprecated, "\
24                                     "instantiate {new} instead."):
25     """
26     Return a "deprecated" class that causes its subclasses to issue a warning.
27     Subclasses of ``new_class`` are considered subclasses of this class.
28     It also warns when the deprecated class is instantiated, but do not when
29     its subclasses are instantiated.
30 
31     It can be used to rename a base class in a library. For example, if we
32     have
33 
34         class OldName(SomeClass):
35             # ...
36 
37     and we want to rename it to NewName, we can do the following::
38 
39         class NewName(SomeClass):
40             # ...
41 
42         OldName = create_deprecated_class('OldName', NewName)
43 
44     Then, if user class inherits from OldName, warning is issued. Also, if
45     some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``
46     checks they'll still return True if sub is a subclass of NewName instead of
47     OldName.
48     """
49 
50     class DeprecatedClass(new_class.__class__):
51 
52         deprecated_class = None
53         warned_on_subclass = False
54 
55         def __new__(metacls, name, bases, clsdict_):
56             cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_)
57             if metacls.deprecated_class is None:
58                 metacls.deprecated_class = cls
59             return cls
60 
61         def __init__(cls, name, bases, clsdict_):
62             meta = cls.__class__
63             old = meta.deprecated_class
64             if old in bases and not (warn_once and meta.warned_on_subclass):
65                 meta.warned_on_subclass = True
66                 msg = subclass_warn_message.format(cls=_clspath(cls),
67                                                    old=_clspath(old, old_class_path),
68                                                    new=_clspath(new_class, new_class_path))
69                 if warn_once:
70                     msg += ' (warning only on first subclass, there may be others)'
71                 warnings.warn(msg, warn_category, stacklevel=2)
72             super(DeprecatedClass, cls).__init__(name, bases, clsdict_)
73 
74         # see http://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
75         # and http://docs.python.org/2/reference/datamodel.html#customizing-instance-and-subclass-checks
76         # for implementation details
77         def __instancecheck__(cls, inst):
78             return any(cls.__subclasscheck__(c)
79                        for c in {type(inst), inst.__class__})
80 
81         def __subclasscheck__(cls, sub):
82             if cls is not DeprecatedClass.deprecated_class:
83                 # we should do the magic only if second `issubclass` argument
84                 # is the deprecated class itself - subclasses of the
85                 # deprecated class should not use custom `__subclasscheck__`
86                 # method.
87                 return super(DeprecatedClass, cls).__subclasscheck__(sub)
88 
89             if not inspect.isclass(sub):
90                 raise TypeError("issubclass() arg 1 must be a class")
91 
92             mro = getattr(sub, '__mro__', ())
93             return any(c in {cls, new_class} for c in mro)
94 
95         def __call__(cls, *args, **kwargs):
96             old = DeprecatedClass.deprecated_class
97             if cls is old:
98                 msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),
99                                                    new=_clspath(new_class, new_class_path))
100                 warnings.warn(msg, warn_category, stacklevel=2)
101             return super(DeprecatedClass, cls).__call__(*args, **kwargs)
102 
103     deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})
104 
105     try:
106         frm = inspect.stack()[1]
107         parent_module = inspect.getmodule(frm[0])
108         if parent_module is not None:
109             deprecated_cls.__module__ = parent_module.__name__
110     except Exception as e:
111         # Sometimes inspect.stack() fails (e.g. when the first import of
112         # deprecated class is in jinja2 template). __module__ attribute is not
113         # important enough to raise an exception as users may be unable
114         # to fix inspect.stack() errors.
115         warnings.warn("Error detecting parent module: %r" % e)
116 
117     return deprecated_cls
118 
119 
120 def _clspath(cls, forced=None):
121     if forced is not None:
122         return forced
123     return '{}.{}'.format(cls.__module__, cls.__name__)
124
```

Path: `scrapy/utils/conf.py`
Content:
```
1 import os
2 import sys
3 from operator import itemgetter
4 
5 import six
6 from six.moves.configparser import SafeConfigParser
7 
8 
9 def build_component_list(base, custom):
10     """Compose a component list based on a custom and base dict of components
11     (typically middlewares or extensions), unless custom is already a list, in
12     which case it's returned.
13     """
14     if isinstance(custom, (list, tuple)):
15         return custom
16     compdict = base.copy()
17     compdict.update(custom)
18     items = (x for x in six.iteritems(compdict) if x[1] is not None)
19     return [x[0] for x in sorted(items, key=itemgetter(1))]
20 
21 
22 def arglist_to_dict(arglist):
23     """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a
24     dict
25     """
26     return dict(x.split('=', 1) for x in arglist)
27 
28 
29 def closest_scrapy_cfg(path='.', prevpath=None):
30     """Return the path to the closest scrapy.cfg file by traversing the current
31     directory and its parents
32     """
33     if path == prevpath:
34         return ''
35     path = os.path.abspath(path)
36     cfgfile = os.path.join(path, 'scrapy.cfg')
37     if os.path.exists(cfgfile):
38         return cfgfile
39     return closest_scrapy_cfg(os.path.dirname(path), path)
40 
41 
42 def init_env(project='default', set_syspath=True):
43     """Initialize environment to use command-line tool from inside a project
44     dir. This sets the Scrapy settings module and modifies the Python path to
45     be able to locate the project module.
46     """
47     cfg = get_config()
48     if cfg.has_option('settings', project):
49         os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project)
50     closest = closest_scrapy_cfg()
51     if closest:
52         projdir = os.path.dirname(closest)
53         if set_syspath and projdir not in sys.path:
54             sys.path.append(projdir)
55 
56 
57 def get_config(use_closest=True):
58     """Get Scrapy config file as a SafeConfigParser"""
59     sources = get_sources(use_closest)
60     cfg = SafeConfigParser()
61     cfg.read(sources)
62     return cfg
63 
64 
65 def get_sources(use_closest=True):
66     xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \
67         os.path.expanduser('~/.config')
68     sources = ['/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg',
69                xdg_config_home + '/scrapy.cfg',
70                os.path.expanduser('~/.scrapy.cfg')]
71     if use_closest:
72         sources.append(closest_scrapy_cfg())
73     return sources
74
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -5,16 +5,30 @@
 import six
 from six.moves.configparser import SafeConfigParser
 
+from scrapy.utils.deprecate import update_classpath
 
-def build_component_list(base, custom):
+
+def build_component_list(base, custom, convert=update_classpath):
     """Compose a component list based on a custom and base dict of components
     (typically middlewares or extensions), unless custom is already a list, in
     which case it's returned.
     """
+
+    def _check_components(complist):
+        if len({convert(c) for c in complist}) != len(complist):
+            raise ValueError('Some paths in {!r} convert to the same object, '
+                             'please update your settings'.format(complist))
+
     if isinstance(custom, (list, tuple)):
-        return custom
-    compdict = base.copy()
-    compdict.update(custom)
+        _check_components(custom)
+        return type(custom)(convert(c) for c in custom)
+
+    def _map_keys(compdict):
+        _check_components(compdict)
+        return {convert(k): v for k, v in six.iteritems(compdict)}
+
+    compdict = _map_keys(base)
+    compdict.update(_map_keys(custom))
     items = (x for x in six.iteritems(compdict) if x[1] is not None)
     return [x[0] for x in sorted(items, key=itemgetter(1))]
 
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -121,3 +121,37 @@
     if forced is not None:
         return forced
     return '{}.{}'.format(cls.__module__, cls.__name__)
+
+
+DEPRECATION_RULES = [
+    ('scrapy.contrib_exp.downloadermiddleware.decompression.', 'scrapy.downloadermiddlewares.decompression.'),
+    ('scrapy.contrib_exp.iterators.', 'scrapy.utils.iterators.'),
+    ('scrapy.contrib.downloadermiddleware.', 'scrapy.downloadermiddlewares.'),
+    ('scrapy.contrib.exporter.', 'scrapy.exporters.'),
+    ('scrapy.contrib.linkextractors.', 'scrapy.linkextractors.'),
+    ('scrapy.contrib.loader.processor.', 'scrapy.loader.processors.'),
+    ('scrapy.contrib.loader.', 'scrapy.loader.'),
+    ('scrapy.contrib.pipeline.', 'scrapy.pipelines.'),
+    ('scrapy.contrib.spidermiddleware.', 'scrapy.spidermiddlewares.'),
+    ('scrapy.contrib.spiders.', 'scrapy.spiders.'),
+    ('scrapy.contrib.', 'scrapy.extensions.'),
+    ('scrapy.command.', 'scrapy.commands.'),
+    ('scrapy.dupefilter.', 'scrapy.dupefilters.'),
+    ('scrapy.linkextractor.', 'scrapy.linkextractors.'),
+    ('scrapy.spider.', 'scrapy.spiders.'),
+    ('scrapy.squeue.', 'scrapy.squeues.'),
+    ('scrapy.statscol.', 'scrapy.statscollectors.'),
+    ('scrapy.utils.decorator.', 'scrapy.utils.decorators.'),
+    ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'),
+]
+
+
+def update_classpath(path):
+    """Update a deprecated path from an object with its new location"""
+    for prefix, replacement in DEPRECATION_RULES:
+        if path.startswith(prefix):
+            new_path = path.replace(prefix, replacement, 1)
+            warnings.warn("`{}` class is deprecated, use `{}` instead".format(path, new_path),
+                          ScrapyDeprecationWarning)
+            return new_path
+    return path

{"golden_diff": "diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py\n--- a/scrapy/utils/conf.py\n+++ b/scrapy/utils/conf.py\n@@ -5,16 +5,30 @@\n import six\n from six.moves.configparser import SafeConfigParser\n \n+from scrapy.utils.deprecate import update_classpath\n \n-def build_component_list(base, custom):\n+\n+def build_component_list(base, custom, convert=update_classpath):\n     \"\"\"Compose a component list based on a custom and base dict of components\n     (typically middlewares or extensions), unless custom is already a list, in\n     which case it's returned.\n     \"\"\"\n+\n+    def _check_components(complist):\n+        if len({convert(c) for c in complist}) != len(complist):\n+            raise ValueError('Some paths in {!r} convert to the same object, '\n+                             'please update your settings'.format(complist))\n+\n     if isinstance(custom, (list, tuple)):\n-        return custom\n-    compdict = base.copy()\n-    compdict.update(custom)\n+        _check_components(custom)\n+        return type(custom)(convert(c) for c in custom)\n+\n+    def _map_keys(compdict):\n+        _check_components(compdict)\n+        return {convert(k): v for k, v in six.iteritems(compdict)}\n+\n+    compdict = _map_keys(base)\n+    compdict.update(_map_keys(custom))\n     items = (x for x in six.iteritems(compdict) if x[1] is not None)\n     return [x[0] for x in sorted(items, key=itemgetter(1))]\n \ndiff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py\n--- a/scrapy/utils/deprecate.py\n+++ b/scrapy/utils/deprecate.py\n@@ -121,3 +121,37 @@\n     if forced is not None:\n         return forced\n     return '{}.{}'.format(cls.__module__, cls.__name__)\n+\n+\n+DEPRECATION_RULES = [\n+    ('scrapy.contrib_exp.downloadermiddleware.decompression.', 'scrapy.downloadermiddlewares.decompression.'),\n+    ('scrapy.contrib_exp.iterators.', 'scrapy.utils.iterators.'),\n+    ('scrapy.contrib.downloadermiddleware.', 'scrapy.downloadermiddlewares.'),\n+    ('scrapy.contrib.exporter.', 'scrapy.exporters.'),\n+    ('scrapy.contrib.linkextractors.', 'scrapy.linkextractors.'),\n+    ('scrapy.contrib.loader.processor.', 'scrapy.loader.processors.'),\n+    ('scrapy.contrib.loader.', 'scrapy.loader.'),\n+    ('scrapy.contrib.pipeline.', 'scrapy.pipelines.'),\n+    ('scrapy.contrib.spidermiddleware.', 'scrapy.spidermiddlewares.'),\n+    ('scrapy.contrib.spiders.', 'scrapy.spiders.'),\n+    ('scrapy.contrib.', 'scrapy.extensions.'),\n+    ('scrapy.command.', 'scrapy.commands.'),\n+    ('scrapy.dupefilter.', 'scrapy.dupefilters.'),\n+    ('scrapy.linkextractor.', 'scrapy.linkextractors.'),\n+    ('scrapy.spider.', 'scrapy.spiders.'),\n+    ('scrapy.squeue.', 'scrapy.squeues.'),\n+    ('scrapy.statscol.', 'scrapy.statscollectors.'),\n+    ('scrapy.utils.decorator.', 'scrapy.utils.decorators.'),\n+    ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'),\n+]\n+\n+\n+def update_classpath(path):\n+    \"\"\"Update a deprecated path from an object with its new location\"\"\"\n+    for prefix, replacement in DEPRECATION_RULES:\n+        if path.startswith(prefix):\n+            new_path = path.replace(prefix, replacement, 1)\n+            warnings.warn(\"`{}` class is deprecated, use `{}` instead\".format(path, new_path),\n+                          ScrapyDeprecationWarning)\n+            return new_path\n+    return path\n", "issue": "Backward incompatibility for relocated paths in settings\nReported by @dangra\n\nThis issue manifests when mixing old paths and new ones for extensions and middlewares (this can happen for example while using a newer version of Scrapy in a project that hasn't updated to the new paths yet). Since paths aren't normalized, the same component can be loaded twice.\n\nTake these settings for example:\n\n``` python\n# scrapy/settings/default_settings.py\nEXTENSIONS_BASE = {\n    'scrapy.extensions.debug.StackTraceDump': 100,  # new path\n} \n```\n\n``` python\n# myproject/settings.py\nEXTENSIONS = {\n    'scrapy.contrib.debug.StackTraceDump': 200,  # old path\n}\n```\n\nWhile merging both dictionaries to build the list of components, the same StackTraceDump class is going to be loaded twice since it appears in two different keys. \n\n", "before_files": [{"content": "\"\"\"Some helpers for deprecation messages\"\"\"\n\nimport warnings\nimport inspect\nfrom scrapy.exceptions import ScrapyDeprecationWarning\n\n\ndef attribute(obj, oldattr, newattr, version='0.12'):\n    cname = obj.__class__.__name__\n    warnings.warn(\"%s.%s attribute is deprecated and will be no longer supported \"\n        \"in Scrapy %s, use %s.%s attribute instead\" % \\\n        (cname, oldattr, version, cname, newattr), ScrapyDeprecationWarning, stacklevel=3)\n\n\ndef create_deprecated_class(name, new_class, clsdict=None,\n                            warn_category=ScrapyDeprecationWarning,\n                            warn_once=True,\n                            old_class_path=None,\n                            new_class_path=None,\n                            subclass_warn_message=\"{cls} inherits from \"\\\n                                    \"deprecated class {old}, please inherit \"\\\n                                    \"from {new}.\",\n                            instance_warn_message=\"{cls} is deprecated, \"\\\n                                    \"instantiate {new} instead.\"):\n    \"\"\"\n    Return a \"deprecated\" class that causes its subclasses to issue a warning.\n    Subclasses of ``new_class`` are considered subclasses of this class.\n    It also warns when the deprecated class is instantiated, but do not when\n    its subclasses are instantiated.\n\n    It can be used to rename a base class in a library. For example, if we\n    have\n\n        class OldName(SomeClass):\n            # ...\n\n    and we want to rename it to NewName, we can do the following::\n\n        class NewName(SomeClass):\n            # ...\n\n        OldName = create_deprecated_class('OldName', NewName)\n\n    Then, if user class inherits from OldName, warning is issued. Also, if\n    some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``\n    checks they'll still return True if sub is a subclass of NewName instead of\n    OldName.\n    \"\"\"\n\n    class DeprecatedClass(new_class.__class__):\n\n        deprecated_class = None\n        warned_on_subclass = False\n\n        def __new__(metacls, name, bases, clsdict_):\n            cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_)\n            if metacls.deprecated_class is None:\n                metacls.deprecated_class = cls\n            return cls\n\n        def __init__(cls, name, bases, clsdict_):\n            meta = cls.__class__\n            old = meta.deprecated_class\n            if old in bases and not (warn_once and meta.warned_on_subclass):\n                meta.warned_on_subclass = True\n                msg = subclass_warn_message.format(cls=_clspath(cls),\n                                                   old=_clspath(old, old_class_path),\n                                                   new=_clspath(new_class, new_class_path))\n                if warn_once:\n                    msg += ' (warning only on first subclass, there may be others)'\n                warnings.warn(msg, warn_category, stacklevel=2)\n            super(DeprecatedClass, cls).__init__(name, bases, clsdict_)\n\n        # see http://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass\n        # and http://docs.python.org/2/reference/datamodel.html#customizing-instance-and-subclass-checks\n        # for implementation details\n        def __instancecheck__(cls, inst):\n            return any(cls.__subclasscheck__(c)\n                       for c in {type(inst), inst.__class__})\n\n        def __subclasscheck__(cls, sub):\n            if cls is not DeprecatedClass.deprecated_class:\n                # we should do the magic only if second `issubclass` argument\n                # is the deprecated class itself - subclasses of the\n                # deprecated class should not use custom `__subclasscheck__`\n                # method.\n                return super(DeprecatedClass, cls).__subclasscheck__(sub)\n\n            if not inspect.isclass(sub):\n                raise TypeError(\"issubclass() arg 1 must be a class\")\n\n            mro = getattr(sub, '__mro__', ())\n            return any(c in {cls, new_class} for c in mro)\n\n        def __call__(cls, *args, **kwargs):\n            old = DeprecatedClass.deprecated_class\n            if cls is old:\n                msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),\n                                                   new=_clspath(new_class, new_class_path))\n                warnings.warn(msg, warn_category, stacklevel=2)\n            return super(DeprecatedClass, cls).__call__(*args, **kwargs)\n\n    deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})\n\n    try:\n        frm = inspect.stack()[1]\n        parent_module = inspect.getmodule(frm[0])\n        if parent_module is not None:\n            deprecated_cls.__module__ = parent_module.__name__\n    except Exception as e:\n        # Sometimes inspect.stack() fails (e.g. when the first import of\n        # deprecated class is in jinja2 template). __module__ attribute is not\n        # important enough to raise an exception as users may be unable\n        # to fix inspect.stack() errors.\n        warnings.warn(\"Error detecting parent module: %r\" % e)\n\n    return deprecated_cls\n\n\ndef _clspath(cls, forced=None):\n    if forced is not None:\n        return forced\n    return '{}.{}'.format(cls.__module__, cls.__name__)\n", "path": "scrapy/utils/deprecate.py"}, {"content": "import os\nimport sys\nfrom operator import itemgetter\n\nimport six\nfrom six.moves.configparser import SafeConfigParser\n\n\ndef build_component_list(base, custom):\n    \"\"\"Compose a component list based on a custom and base dict of components\n    (typically middlewares or extensions), unless custom is already a list, in\n    which case it's returned.\n    \"\"\"\n    if isinstance(custom, (list, tuple)):\n        return custom\n    compdict = base.copy()\n    compdict.update(custom)\n    items = (x for x in six.iteritems(compdict) if x[1] is not None)\n    return [x[0] for x in sorted(items, key=itemgetter(1))]\n\n\ndef arglist_to_dict(arglist):\n    \"\"\"Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a\n    dict\n    \"\"\"\n    return dict(x.split('=', 1) for x in arglist)\n\n\ndef closest_scrapy_cfg(path='.', prevpath=None):\n    \"\"\"Return the path to the closest scrapy.cfg file by traversing the current\n    directory and its parents\n    \"\"\"\n    if path == prevpath:\n        return ''\n    path = os.path.abspath(path)\n    cfgfile = os.path.join(path, 'scrapy.cfg')\n    if os.path.exists(cfgfile):\n        return cfgfile\n    return closest_scrapy_cfg(os.path.dirname(path), path)\n\n\ndef init_env(project='default', set_syspath=True):\n    \"\"\"Initialize environment to use command-line tool from inside a project\n    dir. This sets the Scrapy settings module and modifies the Python path to\n    be able to locate the project module.\n    \"\"\"\n    cfg = get_config()\n    if cfg.has_option('settings', project):\n        os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project)\n    closest = closest_scrapy_cfg()\n    if closest:\n        projdir = os.path.dirname(closest)\n        if set_syspath and projdir not in sys.path:\n            sys.path.append(projdir)\n\n\ndef get_config(use_closest=True):\n    \"\"\"Get Scrapy config file as a SafeConfigParser\"\"\"\n    sources = get_sources(use_closest)\n    cfg = SafeConfigParser()\n    cfg.read(sources)\n    return cfg\n\n\ndef get_sources(use_closest=True):\n    xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \\\n        os.path.expanduser('~/.config')\n    sources = ['/etc/scrapy.cfg', r'c:\\scrapy\\scrapy.cfg',\n               xdg_config_home + '/scrapy.cfg',\n               os.path.expanduser('~/.scrapy.cfg')]\n    if use_closest:\n        sources.append(closest_scrapy_cfg())\n    return sources\n", "path": "scrapy/utils/conf.py"}], "after_files": [{"content": "\"\"\"Some helpers for deprecation messages\"\"\"\n\nimport warnings\nimport inspect\nfrom scrapy.exceptions import ScrapyDeprecationWarning\n\n\ndef attribute(obj, oldattr, newattr, version='0.12'):\n    cname = obj.__class__.__name__\n    warnings.warn(\"%s.%s attribute is deprecated and will be no longer supported \"\n        \"in Scrapy %s, use %s.%s attribute instead\" % \\\n        (cname, oldattr, version, cname, newattr), ScrapyDeprecationWarning, stacklevel=3)\n\n\ndef create_deprecated_class(name, new_class, clsdict=None,\n                            warn_category=ScrapyDeprecationWarning,\n                            warn_once=True,\n                            old_class_path=None,\n                            new_class_path=None,\n                            subclass_warn_message=\"{cls} inherits from \"\\\n                                    \"deprecated class {old}, please inherit \"\\\n                                    \"from {new}.\",\n                            instance_warn_message=\"{cls} is deprecated, \"\\\n                                    \"instantiate {new} instead.\"):\n    \"\"\"\n    Return a \"deprecated\" class that causes its subclasses to issue a warning.\n    Subclasses of ``new_class`` are considered subclasses of this class.\n    It also warns when the deprecated class is instantiated, but do not when\n    its subclasses are instantiated.\n\n    It can be used to rename a base class in a library. For example, if we\n    have\n\n        class OldName(SomeClass):\n            # ...\n\n    and we want to rename it to NewName, we can do the following::\n\n        class NewName(SomeClass):\n            # ...\n\n        OldName = create_deprecated_class('OldName', NewName)\n\n    Then, if user class inherits from OldName, warning is issued. Also, if\n    some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``\n    checks they'll still return True if sub is a subclass of NewName instead of\n    OldName.\n    \"\"\"\n\n    class DeprecatedClass(new_class.__class__):\n\n        deprecated_class = None\n        warned_on_subclass = False\n\n        def __new__(metacls, name, bases, clsdict_):\n            cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_)\n            if metacls.deprecated_class is None:\n                metacls.deprecated_class = cls\n            return cls\n\n        def __init__(cls, name, bases, clsdict_):\n            meta = cls.__class__\n            old = meta.deprecated_class\n            if old in bases and not (warn_once and meta.warned_on_subclass):\n                meta.warned_on_subclass = True\n                msg = subclass_warn_message.format(cls=_clspath(cls),\n                                                   old=_clspath(old, old_class_path),\n                                                   new=_clspath(new_class, new_class_path))\n                if warn_once:\n                    msg += ' (warning only on first subclass, there may be others)'\n                warnings.warn(msg, warn_category, stacklevel=2)\n            super(DeprecatedClass, cls).__init__(name, bases, clsdict_)\n\n        # see http://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass\n        # and http://docs.python.org/2/reference/datamodel.html#customizing-instance-and-subclass-checks\n        # for implementation details\n        def __instancecheck__(cls, inst):\n            return any(cls.__subclasscheck__(c)\n                       for c in {type(inst), inst.__class__})\n\n        def __subclasscheck__(cls, sub):\n            if cls is not DeprecatedClass.deprecated_class:\n                # we should do the magic only if second `issubclass` argument\n                # is the deprecated class itself - subclasses of the\n                # deprecated class should not use custom `__subclasscheck__`\n                # method.\n                return super(DeprecatedClass, cls).__subclasscheck__(sub)\n\n            if not inspect.isclass(sub):\n                raise TypeError(\"issubclass() arg 1 must be a class\")\n\n            mro = getattr(sub, '__mro__', ())\n            return any(c in {cls, new_class} for c in mro)\n\n        def __call__(cls, *args, **kwargs):\n            old = DeprecatedClass.deprecated_class\n            if cls is old:\n                msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),\n                                                   new=_clspath(new_class, new_class_path))\n                warnings.warn(msg, warn_category, stacklevel=2)\n            return super(DeprecatedClass, cls).__call__(*args, **kwargs)\n\n    deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})\n\n    try:\n        frm = inspect.stack()[1]\n        parent_module = inspect.getmodule(frm[0])\n        if parent_module is not None:\n            deprecated_cls.__module__ = parent_module.__name__\n    except Exception as e:\n        # Sometimes inspect.stack() fails (e.g. when the first import of\n        # deprecated class is in jinja2 template). __module__ attribute is not\n        # important enough to raise an exception as users may be unable\n        # to fix inspect.stack() errors.\n        warnings.warn(\"Error detecting parent module: %r\" % e)\n\n    return deprecated_cls\n\n\ndef _clspath(cls, forced=None):\n    if forced is not None:\n        return forced\n    return '{}.{}'.format(cls.__module__, cls.__name__)\n\n\nDEPRECATION_RULES = [\n    ('scrapy.contrib_exp.downloadermiddleware.decompression.', 'scrapy.downloadermiddlewares.decompression.'),\n    ('scrapy.contrib_exp.iterators.', 'scrapy.utils.iterators.'),\n    ('scrapy.contrib.downloadermiddleware.', 'scrapy.downloadermiddlewares.'),\n    ('scrapy.contrib.exporter.', 'scrapy.exporters.'),\n    ('scrapy.contrib.linkextractors.', 'scrapy.linkextractors.'),\n    ('scrapy.contrib.loader.processor.', 'scrapy.loader.processors.'),\n    ('scrapy.contrib.loader.', 'scrapy.loader.'),\n    ('scrapy.contrib.pipeline.', 'scrapy.pipelines.'),\n    ('scrapy.contrib.spidermiddleware.', 'scrapy.spidermiddlewares.'),\n    ('scrapy.contrib.spiders.', 'scrapy.spiders.'),\n    ('scrapy.contrib.', 'scrapy.extensions.'),\n    ('scrapy.command.', 'scrapy.commands.'),\n    ('scrapy.dupefilter.', 'scrapy.dupefilters.'),\n    ('scrapy.linkextractor.', 'scrapy.linkextractors.'),\n    ('scrapy.spider.', 'scrapy.spiders.'),\n    ('scrapy.squeue.', 'scrapy.squeues.'),\n    ('scrapy.statscol.', 'scrapy.statscollectors.'),\n    ('scrapy.utils.decorator.', 'scrapy.utils.decorators.'),\n    ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'),\n]\n\n\ndef update_classpath(path):\n    \"\"\"Update a deprecated path from an object with its new location\"\"\"\n    for prefix, replacement in DEPRECATION_RULES:\n        if path.startswith(prefix):\n            new_path = path.replace(prefix, replacement, 1)\n            warnings.warn(\"`{}` class is deprecated, use `{}` instead\".format(path, new_path),\n                          ScrapyDeprecationWarning)\n            return new_path\n    return path\n", "path": "scrapy/utils/deprecate.py"}, {"content": "import os\nimport sys\nfrom operator import itemgetter\n\nimport six\nfrom six.moves.configparser import SafeConfigParser\n\nfrom scrapy.utils.deprecate import update_classpath\n\n\ndef build_component_list(base, custom, convert=update_classpath):\n    \"\"\"Compose a component list based on a custom and base dict of components\n    (typically middlewares or extensions), unless custom is already a list, in\n    which case it's returned.\n    \"\"\"\n\n    def _check_components(complist):\n        if len({convert(c) for c in complist}) != len(complist):\n            raise ValueError('Some paths in {!r} convert to the same object, '\n                             'please update your settings'.format(complist))\n\n    if isinstance(custom, (list, tuple)):\n        _check_components(custom)\n        return type(custom)(convert(c) for c in custom)\n\n    def _map_keys(compdict):\n        _check_components(compdict)\n        return {convert(k): v for k, v in six.iteritems(compdict)}\n\n    compdict = _map_keys(base)\n    compdict.update(_map_keys(custom))\n    items = (x for x in six.iteritems(compdict) if x[1] is not None)\n    return [x[0] for x in sorted(items, key=itemgetter(1))]\n\n\ndef arglist_to_dict(arglist):\n    \"\"\"Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a\n    dict\n    \"\"\"\n    return dict(x.split('=', 1) for x in arglist)\n\n\ndef closest_scrapy_cfg(path='.', prevpath=None):\n    \"\"\"Return the path to the closest scrapy.cfg file by traversing the current\n    directory and its parents\n    \"\"\"\n    if path == prevpath:\n        return ''\n    path = os.path.abspath(path)\n    cfgfile = os.path.join(path, 'scrapy.cfg')\n    if os.path.exists(cfgfile):\n        return cfgfile\n    return closest_scrapy_cfg(os.path.dirname(path), path)\n\n\ndef init_env(project='default', set_syspath=True):\n    \"\"\"Initialize environment to use command-line tool from inside a project\n    dir. This sets the Scrapy settings module and modifies the Python path to\n    be able to locate the project module.\n    \"\"\"\n    cfg = get_config()\n    if cfg.has_option('settings', project):\n        os.environ['SCRAPY_SETTINGS_MODULE'] = cfg.get('settings', project)\n    closest = closest_scrapy_cfg()\n    if closest:\n        projdir = os.path.dirname(closest)\n        if set_syspath and projdir not in sys.path:\n            sys.path.append(projdir)\n\n\ndef get_config(use_closest=True):\n    \"\"\"Get Scrapy config file as a SafeConfigParser\"\"\"\n    sources = get_sources(use_closest)\n    cfg = SafeConfigParser()\n    cfg.read(sources)\n    return cfg\n\n\ndef get_sources(use_closest=True):\n    xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \\\n        os.path.expanduser('~/.config')\n    sources = ['/etc/scrapy.cfg', r'c:\\scrapy\\scrapy.cfg',\n               xdg_config_home + '/scrapy.cfg',\n               os.path.expanduser('~/.scrapy.cfg')]\n    if use_closest:\n        sources.append(closest_scrapy_cfg())\n    return sources\n", "path": "scrapy/utils/conf.py"}]}