We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Spider.logger not logging custom extra information
I noticed the implicit behavior of the Spider.logger: when logging with extra, extra ultimately do not end up in the log because they are overwritten by default `process` method of [LoggerAdapter](https://github.com/scrapy/scrapy/blob/master/scrapy/spiders/__init__.py#L47)

Current logic:
```py
>>> self.logger.info("test log", extra={"test": "very important information"})
{"message": "test log", "spider": "spider_name"}
```



Expected logic:
```py
>>> self.logger.info("test log", extra={"test": "very important information"})
{"message": "test log", "spider": "spider_name", "test": "very important information"}
```



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/spiders/__init__.py`
Content:
```
1 """
2 Base class for Scrapy spiders
3 
4 See documentation in docs/topics/spiders.rst
5 """
6 
7 from __future__ import annotations
8 
9 import logging
10 from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast
11 
12 from twisted.internet.defer import Deferred
13 
14 from scrapy import signals
15 from scrapy.http import Request, Response
16 from scrapy.utils.trackref import object_ref
17 from scrapy.utils.url import url_is_from_spider
18 
19 if TYPE_CHECKING:
20     # typing.Self requires Python 3.11
21     from typing_extensions import Self
22 
23     from scrapy.crawler import Crawler
24     from scrapy.settings import BaseSettings
25 
26 
27 class Spider(object_ref):
28     """Base class for scrapy spiders. All spiders must inherit from this
29     class.
30     """
31 
32     name: str
33     custom_settings: Optional[dict] = None
34 
35     def __init__(self, name: Optional[str] = None, **kwargs: Any):
36         if name is not None:
37             self.name = name
38         elif not getattr(self, "name", None):
39             raise ValueError(f"{type(self).__name__} must have a name")
40         self.__dict__.update(kwargs)
41         if not hasattr(self, "start_urls"):
42             self.start_urls: List[str] = []
43 
44     @property
45     def logger(self) -> logging.LoggerAdapter:
46         logger = logging.getLogger(self.name)
47         return logging.LoggerAdapter(logger, {"spider": self})
48 
49     def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None:
50         """Log the given message at the given log level
51 
52         This helper wraps a log call to the logger within the spider, but you
53         can use it directly (e.g. Spider.logger.info('msg')) or use any other
54         Python logger too.
55         """
56         self.logger.log(level, message, **kw)
57 
58     @classmethod
59     def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
60         spider = cls(*args, **kwargs)
61         spider._set_crawler(crawler)
62         return spider
63 
64     def _set_crawler(self, crawler: Crawler) -> None:
65         self.crawler = crawler
66         self.settings = crawler.settings
67         crawler.signals.connect(self.close, signals.spider_closed)
68 
69     def start_requests(self) -> Iterable[Request]:
70         if not self.start_urls and hasattr(self, "start_url"):
71             raise AttributeError(
72                 "Crawling could not start: 'start_urls' not found "
73                 "or empty (but found 'start_url' attribute instead, "
74                 "did you miss an 's'?)"
75             )
76         for url in self.start_urls:
77             yield Request(url, dont_filter=True)
78 
79     def _parse(self, response: Response, **kwargs: Any) -> Any:
80         return self.parse(response, **kwargs)
81 
82     def parse(self, response: Response, **kwargs: Any) -> Any:
83         raise NotImplementedError(
84             f"{self.__class__.__name__}.parse callback is not defined"
85         )
86 
87     @classmethod
88     def update_settings(cls, settings: BaseSettings) -> None:
89         settings.setdict(cls.custom_settings or {}, priority="spider")
90 
91     @classmethod
92     def handles_request(cls, request: Request) -> bool:
93         return url_is_from_spider(request.url, cls)
94 
95     @staticmethod
96     def close(spider: Spider, reason: str) -> Union[Deferred, None]:
97         closed = getattr(spider, "closed", None)
98         if callable(closed):
99             return cast(Union[Deferred, None], closed(reason))
100         return None
101 
102     def __repr__(self) -> str:
103         return f"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>"
104 
105 
106 # Top-level imports
107 from scrapy.spiders.crawl import CrawlSpider, Rule
108 from scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider
109 from scrapy.spiders.sitemap import SitemapSpider
110
```

Path: `scrapy/utils/log.py`
Content:
```
1 from __future__ import annotations
2 
3 import logging
4 import sys
5 from logging.config import dictConfig
6 from types import TracebackType
7 from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast
8 
9 from twisted.python import log as twisted_log
10 from twisted.python.failure import Failure
11 
12 import scrapy
13 from scrapy.settings import Settings
14 from scrapy.utils.versions import scrapy_components_versions
15 
16 if TYPE_CHECKING:
17     from scrapy.crawler import Crawler
18 
19 logger = logging.getLogger(__name__)
20 
21 
22 def failure_to_exc_info(
23     failure: Failure,
24 ) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]:
25     """Extract exc_info from Failure instances"""
26     if isinstance(failure, Failure):
27         assert failure.type
28         assert failure.value
29         return (
30             failure.type,
31             failure.value,
32             cast(Optional[TracebackType], failure.getTracebackObject()),
33         )
34     return None
35 
36 
37 class TopLevelFormatter(logging.Filter):
38     """Keep only top level loggers's name (direct children from root) from
39     records.
40 
41     This filter will replace Scrapy loggers' names with 'scrapy'. This mimics
42     the old Scrapy log behaviour and helps shortening long names.
43 
44     Since it can't be set for just one logger (it won't propagate for its
45     children), it's going to be set in the root handler, with a parametrized
46     ``loggers`` list where it should act.
47     """
48 
49     def __init__(self, loggers: Optional[List[str]] = None):
50         self.loggers: List[str] = loggers or []
51 
52     def filter(self, record: logging.LogRecord) -> bool:
53         if any(record.name.startswith(logger + ".") for logger in self.loggers):
54             record.name = record.name.split(".", 1)[0]
55         return True
56 
57 
58 DEFAULT_LOGGING = {
59     "version": 1,
60     "disable_existing_loggers": False,
61     "loggers": {
62         "filelock": {
63             "level": "ERROR",
64         },
65         "hpack": {
66             "level": "ERROR",
67         },
68         "scrapy": {
69             "level": "DEBUG",
70         },
71         "twisted": {
72             "level": "ERROR",
73         },
74     },
75 }
76 
77 
78 def configure_logging(
79     settings: Union[Settings, dict, None] = None, install_root_handler: bool = True
80 ) -> None:
81     """
82     Initialize logging defaults for Scrapy.
83 
84     :param settings: settings used to create and configure a handler for the
85         root logger (default: None).
86     :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None``
87 
88     :param install_root_handler: whether to install root logging handler
89         (default: True)
90     :type install_root_handler: bool
91 
92     This function does:
93 
94     - Route warnings and twisted logging through Python standard logging
95     - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
96     - Route stdout to log if LOG_STDOUT setting is True
97 
98     When ``install_root_handler`` is True (default), this function also
99     creates a handler for the root logger according to given settings
100     (see :ref:`topics-logging-settings`). You can override default options
101     using ``settings`` argument. When ``settings`` is empty or None, defaults
102     are used.
103     """
104     if not sys.warnoptions:
105         # Route warnings through python logging
106         logging.captureWarnings(True)
107 
108     observer = twisted_log.PythonLoggingObserver("twisted")
109     observer.start()
110 
111     dictConfig(DEFAULT_LOGGING)
112 
113     if isinstance(settings, dict) or settings is None:
114         settings = Settings(settings)
115 
116     if settings.getbool("LOG_STDOUT"):
117         sys.stdout = StreamLogger(logging.getLogger("stdout"))  # type: ignore[assignment]
118 
119     if install_root_handler:
120         install_scrapy_root_handler(settings)
121 
122 
123 _scrapy_root_handler: Optional[logging.Handler] = None
124 
125 
126 def install_scrapy_root_handler(settings: Settings) -> None:
127     global _scrapy_root_handler
128 
129     if (
130         _scrapy_root_handler is not None
131         and _scrapy_root_handler in logging.root.handlers
132     ):
133         logging.root.removeHandler(_scrapy_root_handler)
134     logging.root.setLevel(logging.NOTSET)
135     _scrapy_root_handler = _get_handler(settings)
136     logging.root.addHandler(_scrapy_root_handler)
137 
138 
139 def get_scrapy_root_handler() -> Optional[logging.Handler]:
140     return _scrapy_root_handler
141 
142 
143 def _get_handler(settings: Settings) -> logging.Handler:
144     """Return a log handler object according to settings"""
145     filename = settings.get("LOG_FILE")
146     handler: logging.Handler
147     if filename:
148         mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w"
149         encoding = settings.get("LOG_ENCODING")
150         handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
151     elif settings.getbool("LOG_ENABLED"):
152         handler = logging.StreamHandler()
153     else:
154         handler = logging.NullHandler()
155 
156     formatter = logging.Formatter(
157         fmt=settings.get("LOG_FORMAT"), datefmt=settings.get("LOG_DATEFORMAT")
158     )
159     handler.setFormatter(formatter)
160     handler.setLevel(settings.get("LOG_LEVEL"))
161     if settings.getbool("LOG_SHORT_NAMES"):
162         handler.addFilter(TopLevelFormatter(["scrapy"]))
163     return handler
164 
165 
166 def log_scrapy_info(settings: Settings) -> None:
167     logger.info(
168         "Scrapy %(version)s started (bot: %(bot)s)",
169         {"version": scrapy.__version__, "bot": settings["BOT_NAME"]},
170     )
171     versions = [
172         f"{name} {version}"
173         for name, version in scrapy_components_versions()
174         if name != "Scrapy"
175     ]
176     logger.info("Versions: %(versions)s", {"versions": ", ".join(versions)})
177 
178 
179 def log_reactor_info() -> None:
180     from twisted.internet import reactor
181 
182     logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
183     from twisted.internet import asyncioreactor
184 
185     if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
186         logger.debug(
187             "Using asyncio event loop: %s.%s",
188             reactor._asyncioEventloop.__module__,
189             reactor._asyncioEventloop.__class__.__name__,
190         )
191 
192 
193 class StreamLogger:
194     """Fake file-like stream object that redirects writes to a logger instance
195 
196     Taken from:
197         https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
198     """
199 
200     def __init__(self, logger: logging.Logger, log_level: int = logging.INFO):
201         self.logger: logging.Logger = logger
202         self.log_level: int = log_level
203         self.linebuf: str = ""
204 
205     def write(self, buf: str) -> None:
206         for line in buf.rstrip().splitlines():
207             self.logger.log(self.log_level, line.rstrip())
208 
209     def flush(self) -> None:
210         for h in self.logger.handlers:
211             h.flush()
212 
213 
214 class LogCounterHandler(logging.Handler):
215     """Record log levels count into a crawler stats"""
216 
217     def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any):
218         super().__init__(*args, **kwargs)
219         self.crawler: Crawler = crawler
220 
221     def emit(self, record: logging.LogRecord) -> None:
222         sname = f"log_count/{record.levelname}"
223         assert self.crawler.stats
224         self.crawler.stats.inc_value(sname)
225 
226 
227 def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
228     """
229     Helper that takes the dictionary output from the methods in LogFormatter
230     and adapts it into a tuple of positional arguments for logger.log calls,
231     handling backward compatibility as well.
232     """
233 
234     level = logkws.get("level", logging.INFO)
235     message = logkws.get("msg") or ""
236     # NOTE: This also handles 'args' being an empty dict, that case doesn't
237     # play well in logger.log calls
238     args = logkws if not logkws.get("args") else logkws["args"]
239 
240     return (level, message, args)
241
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -22,6 +22,7 @@
 
     from scrapy.crawler import Crawler
     from scrapy.settings import BaseSettings
+    from scrapy.utils.log import SpiderLoggerAdapter
 
 
 class Spider(object_ref):
@@ -42,9 +43,11 @@
             self.start_urls: List[str] = []
 
     @property
-    def logger(self) -> logging.LoggerAdapter:
+    def logger(self) -> SpiderLoggerAdapter:
+        from scrapy.utils.log import SpiderLoggerAdapter
+
         logger = logging.getLogger(self.name)
-        return logging.LoggerAdapter(logger, {"spider": self})
+        return SpiderLoggerAdapter(logger, {"spider": self})
 
     def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None:
         """Log the given message at the given log level
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -4,7 +4,17 @@
 import sys
 from logging.config import dictConfig
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    List,
+    MutableMapping,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+    cast,
+)
 
 from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
@@ -238,3 +248,16 @@
     args = logkws if not logkws.get("args") else logkws["args"]
 
     return (level, message, args)
+
+
+class SpiderLoggerAdapter(logging.LoggerAdapter):
+    def process(
+        self, msg: str, kwargs: MutableMapping[str, Any]
+    ) -> Tuple[str, MutableMapping[str, Any]]:
+        """Method that augments logging with additional 'extra' data"""
+        if isinstance(kwargs.get("extra"), MutableMapping):
+            kwargs["extra"].update(self.extra)
+        else:
+            kwargs["extra"] = self.extra
+
+        return msg, kwargs

{"golden_diff": "diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py\n--- a/scrapy/spiders/__init__.py\n+++ b/scrapy/spiders/__init__.py\n@@ -22,6 +22,7 @@\n \n     from scrapy.crawler import Crawler\n     from scrapy.settings import BaseSettings\n+    from scrapy.utils.log import SpiderLoggerAdapter\n \n \n class Spider(object_ref):\n@@ -42,9 +43,11 @@\n             self.start_urls: List[str] = []\n \n     @property\n-    def logger(self) -> logging.LoggerAdapter:\n+    def logger(self) -> SpiderLoggerAdapter:\n+        from scrapy.utils.log import SpiderLoggerAdapter\n+\n         logger = logging.getLogger(self.name)\n-        return logging.LoggerAdapter(logger, {\"spider\": self})\n+        return SpiderLoggerAdapter(logger, {\"spider\": self})\n \n     def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None:\n         \"\"\"Log the given message at the given log level\ndiff --git a/scrapy/utils/log.py b/scrapy/utils/log.py\n--- a/scrapy/utils/log.py\n+++ b/scrapy/utils/log.py\n@@ -4,7 +4,17 @@\n import sys\n from logging.config import dictConfig\n from types import TracebackType\n-from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast\n+from typing import (\n+    TYPE_CHECKING,\n+    Any,\n+    List,\n+    MutableMapping,\n+    Optional,\n+    Tuple,\n+    Type,\n+    Union,\n+    cast,\n+)\n \n from twisted.python import log as twisted_log\n from twisted.python.failure import Failure\n@@ -238,3 +248,16 @@\n     args = logkws if not logkws.get(\"args\") else logkws[\"args\"]\n \n     return (level, message, args)\n+\n+\n+class SpiderLoggerAdapter(logging.LoggerAdapter):\n+    def process(\n+        self, msg: str, kwargs: MutableMapping[str, Any]\n+    ) -> Tuple[str, MutableMapping[str, Any]]:\n+        \"\"\"Method that augments logging with additional 'extra' data\"\"\"\n+        if isinstance(kwargs.get(\"extra\"), MutableMapping):\n+            kwargs[\"extra\"].update(self.extra)\n+        else:\n+            kwargs[\"extra\"] = self.extra\n+\n+        return msg, kwargs\n", "issue": "Spider.logger not logging custom extra information\nI noticed the implicit behavior of the Spider.logger: when logging with extra, extra ultimately do not end up in the log because they are overwritten by default `process` method of [LoggerAdapter](https://github.com/scrapy/scrapy/blob/master/scrapy/spiders/__init__.py#L47)\r\n\r\nCurrent logic:\r\n```py\r\n>>> self.logger.info(\"test log\", extra={\"test\": \"very important information\"})\r\n{\"message\": \"test log\", \"spider\": \"spider_name\"}\r\n```\r\n\r\n\r\n\r\nExpected logic:\r\n```py\r\n>>> self.logger.info(\"test log\", extra={\"test\": \"very important information\"})\r\n{\"message\": \"test log\", \"spider\": \"spider_name\", \"test\": \"very important information\"}\r\n```\r\n\r\n\n", "before_files": [{"content": "\"\"\"\nBase class for Scrapy spiders\n\nSee documentation in docs/topics/spiders.rst\n\"\"\"\n\nfrom __future__ import annotations\n\nimport logging\nfrom typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast\n\nfrom twisted.internet.defer import Deferred\n\nfrom scrapy import signals\nfrom scrapy.http import Request, Response\nfrom scrapy.utils.trackref import object_ref\nfrom scrapy.utils.url import url_is_from_spider\n\nif TYPE_CHECKING:\n    # typing.Self requires Python 3.11\n    from typing_extensions import Self\n\n    from scrapy.crawler import Crawler\n    from scrapy.settings import BaseSettings\n\n\nclass Spider(object_ref):\n    \"\"\"Base class for scrapy spiders. All spiders must inherit from this\n    class.\n    \"\"\"\n\n    name: str\n    custom_settings: Optional[dict] = None\n\n    def __init__(self, name: Optional[str] = None, **kwargs: Any):\n        if name is not None:\n            self.name = name\n        elif not getattr(self, \"name\", None):\n            raise ValueError(f\"{type(self).__name__} must have a name\")\n        self.__dict__.update(kwargs)\n        if not hasattr(self, \"start_urls\"):\n            self.start_urls: List[str] = []\n\n    @property\n    def logger(self) -> logging.LoggerAdapter:\n        logger = logging.getLogger(self.name)\n        return logging.LoggerAdapter(logger, {\"spider\": self})\n\n    def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None:\n        \"\"\"Log the given message at the given log level\n\n        This helper wraps a log call to the logger within the spider, but you\n        can use it directly (e.g. Spider.logger.info('msg')) or use any other\n        Python logger too.\n        \"\"\"\n        self.logger.log(level, message, **kw)\n\n    @classmethod\n    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:\n        spider = cls(*args, **kwargs)\n        spider._set_crawler(crawler)\n        return spider\n\n    def _set_crawler(self, crawler: Crawler) -> None:\n        self.crawler = crawler\n        self.settings = crawler.settings\n        crawler.signals.connect(self.close, signals.spider_closed)\n\n    def start_requests(self) -> Iterable[Request]:\n        if not self.start_urls and hasattr(self, \"start_url\"):\n            raise AttributeError(\n                \"Crawling could not start: 'start_urls' not found \"\n                \"or empty (but found 'start_url' attribute instead, \"\n                \"did you miss an 's'?)\"\n            )\n        for url in self.start_urls:\n            yield Request(url, dont_filter=True)\n\n    def _parse(self, response: Response, **kwargs: Any) -> Any:\n        return self.parse(response, **kwargs)\n\n    def parse(self, response: Response, **kwargs: Any) -> Any:\n        raise NotImplementedError(\n            f\"{self.__class__.__name__}.parse callback is not defined\"\n        )\n\n    @classmethod\n    def update_settings(cls, settings: BaseSettings) -> None:\n        settings.setdict(cls.custom_settings or {}, priority=\"spider\")\n\n    @classmethod\n    def handles_request(cls, request: Request) -> bool:\n        return url_is_from_spider(request.url, cls)\n\n    @staticmethod\n    def close(spider: Spider, reason: str) -> Union[Deferred, None]:\n        closed = getattr(spider, \"closed\", None)\n        if callable(closed):\n            return cast(Union[Deferred, None], closed(reason))\n        return None\n\n    def __repr__(self) -> str:\n        return f\"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>\"\n\n\n# Top-level imports\nfrom scrapy.spiders.crawl import CrawlSpider, Rule\nfrom scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider\nfrom scrapy.spiders.sitemap import SitemapSpider\n", "path": "scrapy/spiders/__init__.py"}, {"content": "from __future__ import annotations\n\nimport logging\nimport sys\nfrom logging.config import dictConfig\nfrom types import TracebackType\nfrom typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast\n\nfrom twisted.python import log as twisted_log\nfrom twisted.python.failure import Failure\n\nimport scrapy\nfrom scrapy.settings import Settings\nfrom scrapy.utils.versions import scrapy_components_versions\n\nif TYPE_CHECKING:\n    from scrapy.crawler import Crawler\n\nlogger = logging.getLogger(__name__)\n\n\ndef failure_to_exc_info(\n    failure: Failure,\n) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]:\n    \"\"\"Extract exc_info from Failure instances\"\"\"\n    if isinstance(failure, Failure):\n        assert failure.type\n        assert failure.value\n        return (\n            failure.type,\n            failure.value,\n            cast(Optional[TracebackType], failure.getTracebackObject()),\n        )\n    return None\n\n\nclass TopLevelFormatter(logging.Filter):\n    \"\"\"Keep only top level loggers's name (direct children from root) from\n    records.\n\n    This filter will replace Scrapy loggers' names with 'scrapy'. This mimics\n    the old Scrapy log behaviour and helps shortening long names.\n\n    Since it can't be set for just one logger (it won't propagate for its\n    children), it's going to be set in the root handler, with a parametrized\n    ``loggers`` list where it should act.\n    \"\"\"\n\n    def __init__(self, loggers: Optional[List[str]] = None):\n        self.loggers: List[str] = loggers or []\n\n    def filter(self, record: logging.LogRecord) -> bool:\n        if any(record.name.startswith(logger + \".\") for logger in self.loggers):\n            record.name = record.name.split(\".\", 1)[0]\n        return True\n\n\nDEFAULT_LOGGING = {\n    \"version\": 1,\n    \"disable_existing_loggers\": False,\n    \"loggers\": {\n        \"filelock\": {\n            \"level\": \"ERROR\",\n        },\n        \"hpack\": {\n            \"level\": \"ERROR\",\n        },\n        \"scrapy\": {\n            \"level\": \"DEBUG\",\n        },\n        \"twisted\": {\n            \"level\": \"ERROR\",\n        },\n    },\n}\n\n\ndef configure_logging(\n    settings: Union[Settings, dict, None] = None, install_root_handler: bool = True\n) -> None:\n    \"\"\"\n    Initialize logging defaults for Scrapy.\n\n    :param settings: settings used to create and configure a handler for the\n        root logger (default: None).\n    :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None``\n\n    :param install_root_handler: whether to install root logging handler\n        (default: True)\n    :type install_root_handler: bool\n\n    This function does:\n\n    - Route warnings and twisted logging through Python standard logging\n    - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively\n    - Route stdout to log if LOG_STDOUT setting is True\n\n    When ``install_root_handler`` is True (default), this function also\n    creates a handler for the root logger according to given settings\n    (see :ref:`topics-logging-settings`). You can override default options\n    using ``settings`` argument. When ``settings`` is empty or None, defaults\n    are used.\n    \"\"\"\n    if not sys.warnoptions:\n        # Route warnings through python logging\n        logging.captureWarnings(True)\n\n    observer = twisted_log.PythonLoggingObserver(\"twisted\")\n    observer.start()\n\n    dictConfig(DEFAULT_LOGGING)\n\n    if isinstance(settings, dict) or settings is None:\n        settings = Settings(settings)\n\n    if settings.getbool(\"LOG_STDOUT\"):\n        sys.stdout = StreamLogger(logging.getLogger(\"stdout\"))  # type: ignore[assignment]\n\n    if install_root_handler:\n        install_scrapy_root_handler(settings)\n\n\n_scrapy_root_handler: Optional[logging.Handler] = None\n\n\ndef install_scrapy_root_handler(settings: Settings) -> None:\n    global _scrapy_root_handler\n\n    if (\n        _scrapy_root_handler is not None\n        and _scrapy_root_handler in logging.root.handlers\n    ):\n        logging.root.removeHandler(_scrapy_root_handler)\n    logging.root.setLevel(logging.NOTSET)\n    _scrapy_root_handler = _get_handler(settings)\n    logging.root.addHandler(_scrapy_root_handler)\n\n\ndef get_scrapy_root_handler() -> Optional[logging.Handler]:\n    return _scrapy_root_handler\n\n\ndef _get_handler(settings: Settings) -> logging.Handler:\n    \"\"\"Return a log handler object according to settings\"\"\"\n    filename = settings.get(\"LOG_FILE\")\n    handler: logging.Handler\n    if filename:\n        mode = \"a\" if settings.getbool(\"LOG_FILE_APPEND\") else \"w\"\n        encoding = settings.get(\"LOG_ENCODING\")\n        handler = logging.FileHandler(filename, mode=mode, encoding=encoding)\n    elif settings.getbool(\"LOG_ENABLED\"):\n        handler = logging.StreamHandler()\n    else:\n        handler = logging.NullHandler()\n\n    formatter = logging.Formatter(\n        fmt=settings.get(\"LOG_FORMAT\"), datefmt=settings.get(\"LOG_DATEFORMAT\")\n    )\n    handler.setFormatter(formatter)\n    handler.setLevel(settings.get(\"LOG_LEVEL\"))\n    if settings.getbool(\"LOG_SHORT_NAMES\"):\n        handler.addFilter(TopLevelFormatter([\"scrapy\"]))\n    return handler\n\n\ndef log_scrapy_info(settings: Settings) -> None:\n    logger.info(\n        \"Scrapy %(version)s started (bot: %(bot)s)\",\n        {\"version\": scrapy.__version__, \"bot\": settings[\"BOT_NAME\"]},\n    )\n    versions = [\n        f\"{name} {version}\"\n        for name, version in scrapy_components_versions()\n        if name != \"Scrapy\"\n    ]\n    logger.info(\"Versions: %(versions)s\", {\"versions\": \", \".join(versions)})\n\n\ndef log_reactor_info() -> None:\n    from twisted.internet import reactor\n\n    logger.debug(\"Using reactor: %s.%s\", reactor.__module__, reactor.__class__.__name__)\n    from twisted.internet import asyncioreactor\n\n    if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):\n        logger.debug(\n            \"Using asyncio event loop: %s.%s\",\n            reactor._asyncioEventloop.__module__,\n            reactor._asyncioEventloop.__class__.__name__,\n        )\n\n\nclass StreamLogger:\n    \"\"\"Fake file-like stream object that redirects writes to a logger instance\n\n    Taken from:\n        https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/\n    \"\"\"\n\n    def __init__(self, logger: logging.Logger, log_level: int = logging.INFO):\n        self.logger: logging.Logger = logger\n        self.log_level: int = log_level\n        self.linebuf: str = \"\"\n\n    def write(self, buf: str) -> None:\n        for line in buf.rstrip().splitlines():\n            self.logger.log(self.log_level, line.rstrip())\n\n    def flush(self) -> None:\n        for h in self.logger.handlers:\n            h.flush()\n\n\nclass LogCounterHandler(logging.Handler):\n    \"\"\"Record log levels count into a crawler stats\"\"\"\n\n    def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any):\n        super().__init__(*args, **kwargs)\n        self.crawler: Crawler = crawler\n\n    def emit(self, record: logging.LogRecord) -> None:\n        sname = f\"log_count/{record.levelname}\"\n        assert self.crawler.stats\n        self.crawler.stats.inc_value(sname)\n\n\ndef logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:\n    \"\"\"\n    Helper that takes the dictionary output from the methods in LogFormatter\n    and adapts it into a tuple of positional arguments for logger.log calls,\n    handling backward compatibility as well.\n    \"\"\"\n\n    level = logkws.get(\"level\", logging.INFO)\n    message = logkws.get(\"msg\") or \"\"\n    # NOTE: This also handles 'args' being an empty dict, that case doesn't\n    # play well in logger.log calls\n    args = logkws if not logkws.get(\"args\") else logkws[\"args\"]\n\n    return (level, message, args)\n", "path": "scrapy/utils/log.py"}], "after_files": [{"content": "\"\"\"\nBase class for Scrapy spiders\n\nSee documentation in docs/topics/spiders.rst\n\"\"\"\n\nfrom __future__ import annotations\n\nimport logging\nfrom typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast\n\nfrom twisted.internet.defer import Deferred\n\nfrom scrapy import signals\nfrom scrapy.http import Request, Response\nfrom scrapy.utils.trackref import object_ref\nfrom scrapy.utils.url import url_is_from_spider\n\nif TYPE_CHECKING:\n    # typing.Self requires Python 3.11\n    from typing_extensions import Self\n\n    from scrapy.crawler import Crawler\n    from scrapy.settings import BaseSettings\n    from scrapy.utils.log import SpiderLoggerAdapter\n\n\nclass Spider(object_ref):\n    \"\"\"Base class for scrapy spiders. All spiders must inherit from this\n    class.\n    \"\"\"\n\n    name: str\n    custom_settings: Optional[dict] = None\n\n    def __init__(self, name: Optional[str] = None, **kwargs: Any):\n        if name is not None:\n            self.name = name\n        elif not getattr(self, \"name\", None):\n            raise ValueError(f\"{type(self).__name__} must have a name\")\n        self.__dict__.update(kwargs)\n        if not hasattr(self, \"start_urls\"):\n            self.start_urls: List[str] = []\n\n    @property\n    def logger(self) -> SpiderLoggerAdapter:\n        from scrapy.utils.log import SpiderLoggerAdapter\n\n        logger = logging.getLogger(self.name)\n        return SpiderLoggerAdapter(logger, {\"spider\": self})\n\n    def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None:\n        \"\"\"Log the given message at the given log level\n\n        This helper wraps a log call to the logger within the spider, but you\n        can use it directly (e.g. Spider.logger.info('msg')) or use any other\n        Python logger too.\n        \"\"\"\n        self.logger.log(level, message, **kw)\n\n    @classmethod\n    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:\n        spider = cls(*args, **kwargs)\n        spider._set_crawler(crawler)\n        return spider\n\n    def _set_crawler(self, crawler: Crawler) -> None:\n        self.crawler = crawler\n        self.settings = crawler.settings\n        crawler.signals.connect(self.close, signals.spider_closed)\n\n    def start_requests(self) -> Iterable[Request]:\n        if not self.start_urls and hasattr(self, \"start_url\"):\n            raise AttributeError(\n                \"Crawling could not start: 'start_urls' not found \"\n                \"or empty (but found 'start_url' attribute instead, \"\n                \"did you miss an 's'?)\"\n            )\n        for url in self.start_urls:\n            yield Request(url, dont_filter=True)\n\n    def _parse(self, response: Response, **kwargs: Any) -> Any:\n        return self.parse(response, **kwargs)\n\n    def parse(self, response: Response, **kwargs: Any) -> Any:\n        raise NotImplementedError(\n            f\"{self.__class__.__name__}.parse callback is not defined\"\n        )\n\n    @classmethod\n    def update_settings(cls, settings: BaseSettings) -> None:\n        settings.setdict(cls.custom_settings or {}, priority=\"spider\")\n\n    @classmethod\n    def handles_request(cls, request: Request) -> bool:\n        return url_is_from_spider(request.url, cls)\n\n    @staticmethod\n    def close(spider: Spider, reason: str) -> Union[Deferred, None]:\n        closed = getattr(spider, \"closed\", None)\n        if callable(closed):\n            return cast(Union[Deferred, None], closed(reason))\n        return None\n\n    def __repr__(self) -> str:\n        return f\"<{type(self).__name__} {self.name!r} at 0x{id(self):0x}>\"\n\n\n# Top-level imports\nfrom scrapy.spiders.crawl import CrawlSpider, Rule\nfrom scrapy.spiders.feed import CSVFeedSpider, XMLFeedSpider\nfrom scrapy.spiders.sitemap import SitemapSpider\n", "path": "scrapy/spiders/__init__.py"}, {"content": "from __future__ import annotations\n\nimport logging\nimport sys\nfrom logging.config import dictConfig\nfrom types import TracebackType\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    List,\n    MutableMapping,\n    Optional,\n    Tuple,\n    Type,\n    Union,\n    cast,\n)\n\nfrom twisted.python import log as twisted_log\nfrom twisted.python.failure import Failure\n\nimport scrapy\nfrom scrapy.settings import Settings\nfrom scrapy.utils.versions import scrapy_components_versions\n\nif TYPE_CHECKING:\n    from scrapy.crawler import Crawler\n\nlogger = logging.getLogger(__name__)\n\n\ndef failure_to_exc_info(\n    failure: Failure,\n) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]:\n    \"\"\"Extract exc_info from Failure instances\"\"\"\n    if isinstance(failure, Failure):\n        assert failure.type\n        assert failure.value\n        return (\n            failure.type,\n            failure.value,\n            cast(Optional[TracebackType], failure.getTracebackObject()),\n        )\n    return None\n\n\nclass TopLevelFormatter(logging.Filter):\n    \"\"\"Keep only top level loggers's name (direct children from root) from\n    records.\n\n    This filter will replace Scrapy loggers' names with 'scrapy'. This mimics\n    the old Scrapy log behaviour and helps shortening long names.\n\n    Since it can't be set for just one logger (it won't propagate for its\n    children), it's going to be set in the root handler, with a parametrized\n    ``loggers`` list where it should act.\n    \"\"\"\n\n    def __init__(self, loggers: Optional[List[str]] = None):\n        self.loggers: List[str] = loggers or []\n\n    def filter(self, record: logging.LogRecord) -> bool:\n        if any(record.name.startswith(logger + \".\") for logger in self.loggers):\n            record.name = record.name.split(\".\", 1)[0]\n        return True\n\n\nDEFAULT_LOGGING = {\n    \"version\": 1,\n    \"disable_existing_loggers\": False,\n    \"loggers\": {\n        \"filelock\": {\n            \"level\": \"ERROR\",\n        },\n        \"hpack\": {\n            \"level\": \"ERROR\",\n        },\n        \"scrapy\": {\n            \"level\": \"DEBUG\",\n        },\n        \"twisted\": {\n            \"level\": \"ERROR\",\n        },\n    },\n}\n\n\ndef configure_logging(\n    settings: Union[Settings, dict, None] = None, install_root_handler: bool = True\n) -> None:\n    \"\"\"\n    Initialize logging defaults for Scrapy.\n\n    :param settings: settings used to create and configure a handler for the\n        root logger (default: None).\n    :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None``\n\n    :param install_root_handler: whether to install root logging handler\n        (default: True)\n    :type install_root_handler: bool\n\n    This function does:\n\n    - Route warnings and twisted logging through Python standard logging\n    - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively\n    - Route stdout to log if LOG_STDOUT setting is True\n\n    When ``install_root_handler`` is True (default), this function also\n    creates a handler for the root logger according to given settings\n    (see :ref:`topics-logging-settings`). You can override default options\n    using ``settings`` argument. When ``settings`` is empty or None, defaults\n    are used.\n    \"\"\"\n    if not sys.warnoptions:\n        # Route warnings through python logging\n        logging.captureWarnings(True)\n\n    observer = twisted_log.PythonLoggingObserver(\"twisted\")\n    observer.start()\n\n    dictConfig(DEFAULT_LOGGING)\n\n    if isinstance(settings, dict) or settings is None:\n        settings = Settings(settings)\n\n    if settings.getbool(\"LOG_STDOUT\"):\n        sys.stdout = StreamLogger(logging.getLogger(\"stdout\"))  # type: ignore[assignment]\n\n    if install_root_handler:\n        install_scrapy_root_handler(settings)\n\n\n_scrapy_root_handler: Optional[logging.Handler] = None\n\n\ndef install_scrapy_root_handler(settings: Settings) -> None:\n    global _scrapy_root_handler\n\n    if (\n        _scrapy_root_handler is not None\n        and _scrapy_root_handler in logging.root.handlers\n    ):\n        logging.root.removeHandler(_scrapy_root_handler)\n    logging.root.setLevel(logging.NOTSET)\n    _scrapy_root_handler = _get_handler(settings)\n    logging.root.addHandler(_scrapy_root_handler)\n\n\ndef get_scrapy_root_handler() -> Optional[logging.Handler]:\n    return _scrapy_root_handler\n\n\ndef _get_handler(settings: Settings) -> logging.Handler:\n    \"\"\"Return a log handler object according to settings\"\"\"\n    filename = settings.get(\"LOG_FILE\")\n    handler: logging.Handler\n    if filename:\n        mode = \"a\" if settings.getbool(\"LOG_FILE_APPEND\") else \"w\"\n        encoding = settings.get(\"LOG_ENCODING\")\n        handler = logging.FileHandler(filename, mode=mode, encoding=encoding)\n    elif settings.getbool(\"LOG_ENABLED\"):\n        handler = logging.StreamHandler()\n    else:\n        handler = logging.NullHandler()\n\n    formatter = logging.Formatter(\n        fmt=settings.get(\"LOG_FORMAT\"), datefmt=settings.get(\"LOG_DATEFORMAT\")\n    )\n    handler.setFormatter(formatter)\n    handler.setLevel(settings.get(\"LOG_LEVEL\"))\n    if settings.getbool(\"LOG_SHORT_NAMES\"):\n        handler.addFilter(TopLevelFormatter([\"scrapy\"]))\n    return handler\n\n\ndef log_scrapy_info(settings: Settings) -> None:\n    logger.info(\n        \"Scrapy %(version)s started (bot: %(bot)s)\",\n        {\"version\": scrapy.__version__, \"bot\": settings[\"BOT_NAME\"]},\n    )\n    versions = [\n        f\"{name} {version}\"\n        for name, version in scrapy_components_versions()\n        if name != \"Scrapy\"\n    ]\n    logger.info(\"Versions: %(versions)s\", {\"versions\": \", \".join(versions)})\n\n\ndef log_reactor_info() -> None:\n    from twisted.internet import reactor\n\n    logger.debug(\"Using reactor: %s.%s\", reactor.__module__, reactor.__class__.__name__)\n    from twisted.internet import asyncioreactor\n\n    if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):\n        logger.debug(\n            \"Using asyncio event loop: %s.%s\",\n            reactor._asyncioEventloop.__module__,\n            reactor._asyncioEventloop.__class__.__name__,\n        )\n\n\nclass StreamLogger:\n    \"\"\"Fake file-like stream object that redirects writes to a logger instance\n\n    Taken from:\n        https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/\n    \"\"\"\n\n    def __init__(self, logger: logging.Logger, log_level: int = logging.INFO):\n        self.logger: logging.Logger = logger\n        self.log_level: int = log_level\n        self.linebuf: str = \"\"\n\n    def write(self, buf: str) -> None:\n        for line in buf.rstrip().splitlines():\n            self.logger.log(self.log_level, line.rstrip())\n\n    def flush(self) -> None:\n        for h in self.logger.handlers:\n            h.flush()\n\n\nclass LogCounterHandler(logging.Handler):\n    \"\"\"Record log levels count into a crawler stats\"\"\"\n\n    def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any):\n        super().__init__(*args, **kwargs)\n        self.crawler: Crawler = crawler\n\n    def emit(self, record: logging.LogRecord) -> None:\n        sname = f\"log_count/{record.levelname}\"\n        assert self.crawler.stats\n        self.crawler.stats.inc_value(sname)\n\n\ndef logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:\n    \"\"\"\n    Helper that takes the dictionary output from the methods in LogFormatter\n    and adapts it into a tuple of positional arguments for logger.log calls,\n    handling backward compatibility as well.\n    \"\"\"\n\n    level = logkws.get(\"level\", logging.INFO)\n    message = logkws.get(\"msg\") or \"\"\n    # NOTE: This also handles 'args' being an empty dict, that case doesn't\n    # play well in logger.log calls\n    args = logkws if not logkws.get(\"args\") else logkws[\"args\"]\n\n    return (level, message, args)\n\n\nclass SpiderLoggerAdapter(logging.LoggerAdapter):\n    def process(\n        self, msg: str, kwargs: MutableMapping[str, Any]\n    ) -> Tuple[str, MutableMapping[str, Any]]:\n        \"\"\"Method that augments logging with additional 'extra' data\"\"\"\n        if isinstance(kwargs.get(\"extra\"), MutableMapping):\n            kwargs[\"extra\"].update(self.extra)\n        else:\n            kwargs[\"extra\"] = self.extra\n\n        return msg, kwargs\n", "path": "scrapy/utils/log.py"}]}