We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Remove deprecated scrapy.utils.response.response_httprepr
Deprecated in 2.6.0.
Remove deprecated support from scrapy.utils.log.logformatter_adapter()
This helper was added in 1.0.0 (I think), it looks like the parts of it that don't show warnings are still useful, though I haven't looked closely at them.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/utils/log.py`
Content:
```
1 from __future__ import annotations
2 
3 import logging
4 import sys
5 import warnings
6 from logging.config import dictConfig
7 from types import TracebackType
8 from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast
9 
10 from twisted.python import log as twisted_log
11 from twisted.python.failure import Failure
12 
13 import scrapy
14 from scrapy.exceptions import ScrapyDeprecationWarning
15 from scrapy.settings import Settings
16 from scrapy.utils.versions import scrapy_components_versions
17 
18 if TYPE_CHECKING:
19     from scrapy.crawler import Crawler
20 
21 logger = logging.getLogger(__name__)
22 
23 
24 def failure_to_exc_info(
25     failure: Failure,
26 ) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]:
27     """Extract exc_info from Failure instances"""
28     if isinstance(failure, Failure):
29         assert failure.type
30         assert failure.value
31         return (
32             failure.type,
33             failure.value,
34             cast(Optional[TracebackType], failure.getTracebackObject()),
35         )
36     return None
37 
38 
39 class TopLevelFormatter(logging.Filter):
40     """Keep only top level loggers's name (direct children from root) from
41     records.
42 
43     This filter will replace Scrapy loggers' names with 'scrapy'. This mimics
44     the old Scrapy log behaviour and helps shortening long names.
45 
46     Since it can't be set for just one logger (it won't propagate for its
47     children), it's going to be set in the root handler, with a parametrized
48     ``loggers`` list where it should act.
49     """
50 
51     def __init__(self, loggers: Optional[List[str]] = None):
52         self.loggers: List[str] = loggers or []
53 
54     def filter(self, record: logging.LogRecord) -> bool:
55         if any(record.name.startswith(logger + ".") for logger in self.loggers):
56             record.name = record.name.split(".", 1)[0]
57         return True
58 
59 
60 DEFAULT_LOGGING = {
61     "version": 1,
62     "disable_existing_loggers": False,
63     "loggers": {
64         "filelock": {
65             "level": "ERROR",
66         },
67         "hpack": {
68             "level": "ERROR",
69         },
70         "scrapy": {
71             "level": "DEBUG",
72         },
73         "twisted": {
74             "level": "ERROR",
75         },
76     },
77 }
78 
79 
80 def configure_logging(
81     settings: Union[Settings, dict, None] = None, install_root_handler: bool = True
82 ) -> None:
83     """
84     Initialize logging defaults for Scrapy.
85 
86     :param settings: settings used to create and configure a handler for the
87         root logger (default: None).
88     :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None``
89 
90     :param install_root_handler: whether to install root logging handler
91         (default: True)
92     :type install_root_handler: bool
93 
94     This function does:
95 
96     - Route warnings and twisted logging through Python standard logging
97     - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
98     - Route stdout to log if LOG_STDOUT setting is True
99 
100     When ``install_root_handler`` is True (default), this function also
101     creates a handler for the root logger according to given settings
102     (see :ref:`topics-logging-settings`). You can override default options
103     using ``settings`` argument. When ``settings`` is empty or None, defaults
104     are used.
105     """
106     if not sys.warnoptions:
107         # Route warnings through python logging
108         logging.captureWarnings(True)
109 
110     observer = twisted_log.PythonLoggingObserver("twisted")
111     observer.start()
112 
113     dictConfig(DEFAULT_LOGGING)
114 
115     if isinstance(settings, dict) or settings is None:
116         settings = Settings(settings)
117 
118     if settings.getbool("LOG_STDOUT"):
119         sys.stdout = StreamLogger(logging.getLogger("stdout"))  # type: ignore[assignment]
120 
121     if install_root_handler:
122         install_scrapy_root_handler(settings)
123 
124 
125 _scrapy_root_handler: Optional[logging.Handler] = None
126 
127 
128 def install_scrapy_root_handler(settings: Settings) -> None:
129     global _scrapy_root_handler
130 
131     if (
132         _scrapy_root_handler is not None
133         and _scrapy_root_handler in logging.root.handlers
134     ):
135         logging.root.removeHandler(_scrapy_root_handler)
136     logging.root.setLevel(logging.NOTSET)
137     _scrapy_root_handler = _get_handler(settings)
138     logging.root.addHandler(_scrapy_root_handler)
139 
140 
141 def get_scrapy_root_handler() -> Optional[logging.Handler]:
142     return _scrapy_root_handler
143 
144 
145 def _get_handler(settings: Settings) -> logging.Handler:
146     """Return a log handler object according to settings"""
147     filename = settings.get("LOG_FILE")
148     handler: logging.Handler
149     if filename:
150         mode = "a" if settings.getbool("LOG_FILE_APPEND") else "w"
151         encoding = settings.get("LOG_ENCODING")
152         handler = logging.FileHandler(filename, mode=mode, encoding=encoding)
153     elif settings.getbool("LOG_ENABLED"):
154         handler = logging.StreamHandler()
155     else:
156         handler = logging.NullHandler()
157 
158     formatter = logging.Formatter(
159         fmt=settings.get("LOG_FORMAT"), datefmt=settings.get("LOG_DATEFORMAT")
160     )
161     handler.setFormatter(formatter)
162     handler.setLevel(settings.get("LOG_LEVEL"))
163     if settings.getbool("LOG_SHORT_NAMES"):
164         handler.addFilter(TopLevelFormatter(["scrapy"]))
165     return handler
166 
167 
168 def log_scrapy_info(settings: Settings) -> None:
169     logger.info(
170         "Scrapy %(version)s started (bot: %(bot)s)",
171         {"version": scrapy.__version__, "bot": settings["BOT_NAME"]},
172     )
173     versions = [
174         f"{name} {version}"
175         for name, version in scrapy_components_versions()
176         if name != "Scrapy"
177     ]
178     logger.info("Versions: %(versions)s", {"versions": ", ".join(versions)})
179 
180 
181 def log_reactor_info() -> None:
182     from twisted.internet import reactor
183 
184     logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
185     from twisted.internet import asyncioreactor
186 
187     if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
188         logger.debug(
189             "Using asyncio event loop: %s.%s",
190             reactor._asyncioEventloop.__module__,
191             reactor._asyncioEventloop.__class__.__name__,
192         )
193 
194 
195 class StreamLogger:
196     """Fake file-like stream object that redirects writes to a logger instance
197 
198     Taken from:
199         https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
200     """
201 
202     def __init__(self, logger: logging.Logger, log_level: int = logging.INFO):
203         self.logger: logging.Logger = logger
204         self.log_level: int = log_level
205         self.linebuf: str = ""
206 
207     def write(self, buf: str) -> None:
208         for line in buf.rstrip().splitlines():
209             self.logger.log(self.log_level, line.rstrip())
210 
211     def flush(self) -> None:
212         for h in self.logger.handlers:
213             h.flush()
214 
215 
216 class LogCounterHandler(logging.Handler):
217     """Record log levels count into a crawler stats"""
218 
219     def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any):
220         super().__init__(*args, **kwargs)
221         self.crawler: Crawler = crawler
222 
223     def emit(self, record: logging.LogRecord) -> None:
224         sname = f"log_count/{record.levelname}"
225         assert self.crawler.stats
226         self.crawler.stats.inc_value(sname)
227 
228 
229 def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
230     """
231     Helper that takes the dictionary output from the methods in LogFormatter
232     and adapts it into a tuple of positional arguments for logger.log calls,
233     handling backward compatibility as well.
234     """
235     if not {"level", "msg", "args"} <= set(logkws):
236         warnings.warn("Missing keys in LogFormatter method", ScrapyDeprecationWarning)
237 
238     if "format" in logkws:
239         warnings.warn(
240             "`format` key in LogFormatter methods has been "
241             "deprecated, use `msg` instead",
242             ScrapyDeprecationWarning,
243         )
244 
245     level = logkws.get("level", logging.INFO)
246     message = logkws.get("format", logkws.get("msg"))
247     # NOTE: This also handles 'args' being an empty dict, that case doesn't
248     # play well in logger.log calls
249     args = logkws if not logkws.get("args") else logkws["args"]
250 
251     return (level, message, args)
252
```

Path: `scrapy/utils/response.py`
Content:
```
1 """
2 This module provides some useful functions for working with
3 scrapy.http.Response objects
4 """
5 import os
6 import re
7 import tempfile
8 import webbrowser
9 from typing import Any, Callable, Iterable, Tuple, Union
10 from weakref import WeakKeyDictionary
11 
12 from twisted.web import http
13 from w3lib import html
14 
15 import scrapy
16 from scrapy.http.response import Response
17 from scrapy.utils.decorators import deprecated
18 from scrapy.utils.python import to_bytes, to_unicode
19 
20 _baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
21 
22 
23 def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
24     """Return the base url of the given response, joined with the response url"""
25     if response not in _baseurl_cache:
26         text = response.text[0:4096]
27         _baseurl_cache[response] = html.get_base_url(
28             text, response.url, response.encoding
29         )
30     return _baseurl_cache[response]
31 
32 
33 _metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = (
34     WeakKeyDictionary()
35 )
36 
37 
38 def get_meta_refresh(
39     response: "scrapy.http.response.text.TextResponse",
40     ignore_tags: Iterable[str] = ("script", "noscript"),
41 ) -> Union[Tuple[None, None], Tuple[float, str]]:
42     """Parse the http-equiv refresh parameter from the given response"""
43     if response not in _metaref_cache:
44         text = response.text[0:4096]
45         _metaref_cache[response] = html.get_meta_refresh(
46             text, response.url, response.encoding, ignore_tags=ignore_tags
47         )
48     return _metaref_cache[response]
49 
50 
51 def response_status_message(status: Union[bytes, float, int, str]) -> str:
52     """Return status code plus status text descriptive message"""
53     status_int = int(status)
54     message = http.RESPONSES.get(status_int, "Unknown Status")
55     return f"{status_int} {to_unicode(message)}"
56 
57 
58 @deprecated
59 def response_httprepr(response: Response) -> bytes:
60     """Return raw HTTP representation (as bytes) of the given response. This
61     is provided only for reference, since it's not the exact stream of bytes
62     that was received (that's not exposed by Twisted).
63     """
64     values = [
65         b"HTTP/1.1 ",
66         to_bytes(str(response.status)),
67         b" ",
68         to_bytes(http.RESPONSES.get(response.status, b"")),
69         b"\r\n",
70     ]
71     if response.headers:
72         values.extend([response.headers.to_string(), b"\r\n"])
73     values.extend([b"\r\n", response.body])
74     return b"".join(values)
75 
76 
77 def open_in_browser(
78     response: Union[
79         "scrapy.http.response.html.HtmlResponse",
80         "scrapy.http.response.text.TextResponse",
81     ],
82     _openfunc: Callable[[str], Any] = webbrowser.open,
83 ) -> Any:
84     """Open the given response in a local web browser, populating the <base>
85     tag for external links to work
86     """
87     from scrapy.http import HtmlResponse, TextResponse
88 
89     # XXX: this implementation is a bit dirty and could be improved
90     body = response.body
91     if isinstance(response, HtmlResponse):
92         if b"<base" not in body:
93             repl = rf'\1<base href="{response.url}">'
94             body = re.sub(b"<!--.*?-->", b"", body, flags=re.DOTALL)
95             body = re.sub(rb"(<head(?:>|\s.*?>))", to_bytes(repl), body)
96         ext = ".html"
97     elif isinstance(response, TextResponse):
98         ext = ".txt"
99     else:
100         raise TypeError("Unsupported response type: " f"{response.__class__.__name__}")
101     fd, fname = tempfile.mkstemp(ext)
102     os.write(fd, body)
103     os.close(fd)
104     return _openfunc(f"file://{fname}")
105
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -2,7 +2,6 @@
 
 import logging
 import sys
-import warnings
 from logging.config import dictConfig
 from types import TracebackType
 from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast
@@ -11,7 +10,6 @@
 from twisted.python.failure import Failure
 
 import scrapy
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
 from scrapy.utils.versions import scrapy_components_versions
 
@@ -232,18 +230,9 @@
     and adapts it into a tuple of positional arguments for logger.log calls,
     handling backward compatibility as well.
     """
-    if not {"level", "msg", "args"} <= set(logkws):
-        warnings.warn("Missing keys in LogFormatter method", ScrapyDeprecationWarning)
-
-    if "format" in logkws:
-        warnings.warn(
-            "`format` key in LogFormatter methods has been "
-            "deprecated, use `msg` instead",
-            ScrapyDeprecationWarning,
-        )
 
     level = logkws.get("level", logging.INFO)
-    message = logkws.get("format", logkws.get("msg"))
+    message = logkws.get("msg") or ""
     # NOTE: This also handles 'args' being an empty dict, that case doesn't
     # play well in logger.log calls
     args = logkws if not logkws.get("args") else logkws["args"]
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -14,7 +14,6 @@
 
 import scrapy
 from scrapy.http.response import Response
-from scrapy.utils.decorators import deprecated
 from scrapy.utils.python import to_bytes, to_unicode
 
 _baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
@@ -55,25 +54,6 @@
     return f"{status_int} {to_unicode(message)}"
 
 
-@deprecated
-def response_httprepr(response: Response) -> bytes:
-    """Return raw HTTP representation (as bytes) of the given response. This
-    is provided only for reference, since it's not the exact stream of bytes
-    that was received (that's not exposed by Twisted).
-    """
-    values = [
-        b"HTTP/1.1 ",
-        to_bytes(str(response.status)),
-        b" ",
-        to_bytes(http.RESPONSES.get(response.status, b"")),
-        b"\r\n",
-    ]
-    if response.headers:
-        values.extend([response.headers.to_string(), b"\r\n"])
-    values.extend([b"\r\n", response.body])
-    return b"".join(values)
-
-
 def open_in_browser(
     response: Union[
         "scrapy.http.response.html.HtmlResponse",

{"golden_diff": "diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py\n--- a/scrapy/utils/log.py\n+++ b/scrapy/utils/log.py\n@@ -2,7 +2,6 @@\n \n import logging\n import sys\n-import warnings\n from logging.config import dictConfig\n from types import TracebackType\n from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast\n@@ -11,7 +10,6 @@\n from twisted.python.failure import Failure\n \n import scrapy\n-from scrapy.exceptions import ScrapyDeprecationWarning\n from scrapy.settings import Settings\n from scrapy.utils.versions import scrapy_components_versions\n \n@@ -232,18 +230,9 @@\n     and adapts it into a tuple of positional arguments for logger.log calls,\n     handling backward compatibility as well.\n     \"\"\"\n-    if not {\"level\", \"msg\", \"args\"} <= set(logkws):\n-        warnings.warn(\"Missing keys in LogFormatter method\", ScrapyDeprecationWarning)\n-\n-    if \"format\" in logkws:\n-        warnings.warn(\n-            \"`format` key in LogFormatter methods has been \"\n-            \"deprecated, use `msg` instead\",\n-            ScrapyDeprecationWarning,\n-        )\n \n     level = logkws.get(\"level\", logging.INFO)\n-    message = logkws.get(\"format\", logkws.get(\"msg\"))\n+    message = logkws.get(\"msg\") or \"\"\n     # NOTE: This also handles 'args' being an empty dict, that case doesn't\n     # play well in logger.log calls\n     args = logkws if not logkws.get(\"args\") else logkws[\"args\"]\ndiff --git a/scrapy/utils/response.py b/scrapy/utils/response.py\n--- a/scrapy/utils/response.py\n+++ b/scrapy/utils/response.py\n@@ -14,7 +14,6 @@\n \n import scrapy\n from scrapy.http.response import Response\n-from scrapy.utils.decorators import deprecated\n from scrapy.utils.python import to_bytes, to_unicode\n \n _baseurl_cache: \"WeakKeyDictionary[Response, str]\" = WeakKeyDictionary()\n@@ -55,25 +54,6 @@\n     return f\"{status_int} {to_unicode(message)}\"\n \n \n-@deprecated\n-def response_httprepr(response: Response) -> bytes:\n-    \"\"\"Return raw HTTP representation (as bytes) of the given response. This\n-    is provided only for reference, since it's not the exact stream of bytes\n-    that was received (that's not exposed by Twisted).\n-    \"\"\"\n-    values = [\n-        b\"HTTP/1.1 \",\n-        to_bytes(str(response.status)),\n-        b\" \",\n-        to_bytes(http.RESPONSES.get(response.status, b\"\")),\n-        b\"\\r\\n\",\n-    ]\n-    if response.headers:\n-        values.extend([response.headers.to_string(), b\"\\r\\n\"])\n-    values.extend([b\"\\r\\n\", response.body])\n-    return b\"\".join(values)\n-\n-\n def open_in_browser(\n     response: Union[\n         \"scrapy.http.response.html.HtmlResponse\",\n", "issue": "Remove deprecated scrapy.utils.response.response_httprepr\nDeprecated in 2.6.0.\nRemove deprecated support from scrapy.utils.log.logformatter_adapter()\nThis helper was added in 1.0.0 (I think), it looks like the parts of it that don't show warnings are still useful, though I haven't looked closely at them.\n", "before_files": [{"content": "from __future__ import annotations\n\nimport logging\nimport sys\nimport warnings\nfrom logging.config import dictConfig\nfrom types import TracebackType\nfrom typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast\n\nfrom twisted.python import log as twisted_log\nfrom twisted.python.failure import Failure\n\nimport scrapy\nfrom scrapy.exceptions import ScrapyDeprecationWarning\nfrom scrapy.settings import Settings\nfrom scrapy.utils.versions import scrapy_components_versions\n\nif TYPE_CHECKING:\n    from scrapy.crawler import Crawler\n\nlogger = logging.getLogger(__name__)\n\n\ndef failure_to_exc_info(\n    failure: Failure,\n) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]:\n    \"\"\"Extract exc_info from Failure instances\"\"\"\n    if isinstance(failure, Failure):\n        assert failure.type\n        assert failure.value\n        return (\n            failure.type,\n            failure.value,\n            cast(Optional[TracebackType], failure.getTracebackObject()),\n        )\n    return None\n\n\nclass TopLevelFormatter(logging.Filter):\n    \"\"\"Keep only top level loggers's name (direct children from root) from\n    records.\n\n    This filter will replace Scrapy loggers' names with 'scrapy'. This mimics\n    the old Scrapy log behaviour and helps shortening long names.\n\n    Since it can't be set for just one logger (it won't propagate for its\n    children), it's going to be set in the root handler, with a parametrized\n    ``loggers`` list where it should act.\n    \"\"\"\n\n    def __init__(self, loggers: Optional[List[str]] = None):\n        self.loggers: List[str] = loggers or []\n\n    def filter(self, record: logging.LogRecord) -> bool:\n        if any(record.name.startswith(logger + \".\") for logger in self.loggers):\n            record.name = record.name.split(\".\", 1)[0]\n        return True\n\n\nDEFAULT_LOGGING = {\n    \"version\": 1,\n    \"disable_existing_loggers\": False,\n    \"loggers\": {\n        \"filelock\": {\n            \"level\": \"ERROR\",\n        },\n        \"hpack\": {\n            \"level\": \"ERROR\",\n        },\n        \"scrapy\": {\n            \"level\": \"DEBUG\",\n        },\n        \"twisted\": {\n            \"level\": \"ERROR\",\n        },\n    },\n}\n\n\ndef configure_logging(\n    settings: Union[Settings, dict, None] = None, install_root_handler: bool = True\n) -> None:\n    \"\"\"\n    Initialize logging defaults for Scrapy.\n\n    :param settings: settings used to create and configure a handler for the\n        root logger (default: None).\n    :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None``\n\n    :param install_root_handler: whether to install root logging handler\n        (default: True)\n    :type install_root_handler: bool\n\n    This function does:\n\n    - Route warnings and twisted logging through Python standard logging\n    - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively\n    - Route stdout to log if LOG_STDOUT setting is True\n\n    When ``install_root_handler`` is True (default), this function also\n    creates a handler for the root logger according to given settings\n    (see :ref:`topics-logging-settings`). You can override default options\n    using ``settings`` argument. When ``settings`` is empty or None, defaults\n    are used.\n    \"\"\"\n    if not sys.warnoptions:\n        # Route warnings through python logging\n        logging.captureWarnings(True)\n\n    observer = twisted_log.PythonLoggingObserver(\"twisted\")\n    observer.start()\n\n    dictConfig(DEFAULT_LOGGING)\n\n    if isinstance(settings, dict) or settings is None:\n        settings = Settings(settings)\n\n    if settings.getbool(\"LOG_STDOUT\"):\n        sys.stdout = StreamLogger(logging.getLogger(\"stdout\"))  # type: ignore[assignment]\n\n    if install_root_handler:\n        install_scrapy_root_handler(settings)\n\n\n_scrapy_root_handler: Optional[logging.Handler] = None\n\n\ndef install_scrapy_root_handler(settings: Settings) -> None:\n    global _scrapy_root_handler\n\n    if (\n        _scrapy_root_handler is not None\n        and _scrapy_root_handler in logging.root.handlers\n    ):\n        logging.root.removeHandler(_scrapy_root_handler)\n    logging.root.setLevel(logging.NOTSET)\n    _scrapy_root_handler = _get_handler(settings)\n    logging.root.addHandler(_scrapy_root_handler)\n\n\ndef get_scrapy_root_handler() -> Optional[logging.Handler]:\n    return _scrapy_root_handler\n\n\ndef _get_handler(settings: Settings) -> logging.Handler:\n    \"\"\"Return a log handler object according to settings\"\"\"\n    filename = settings.get(\"LOG_FILE\")\n    handler: logging.Handler\n    if filename:\n        mode = \"a\" if settings.getbool(\"LOG_FILE_APPEND\") else \"w\"\n        encoding = settings.get(\"LOG_ENCODING\")\n        handler = logging.FileHandler(filename, mode=mode, encoding=encoding)\n    elif settings.getbool(\"LOG_ENABLED\"):\n        handler = logging.StreamHandler()\n    else:\n        handler = logging.NullHandler()\n\n    formatter = logging.Formatter(\n        fmt=settings.get(\"LOG_FORMAT\"), datefmt=settings.get(\"LOG_DATEFORMAT\")\n    )\n    handler.setFormatter(formatter)\n    handler.setLevel(settings.get(\"LOG_LEVEL\"))\n    if settings.getbool(\"LOG_SHORT_NAMES\"):\n        handler.addFilter(TopLevelFormatter([\"scrapy\"]))\n    return handler\n\n\ndef log_scrapy_info(settings: Settings) -> None:\n    logger.info(\n        \"Scrapy %(version)s started (bot: %(bot)s)\",\n        {\"version\": scrapy.__version__, \"bot\": settings[\"BOT_NAME\"]},\n    )\n    versions = [\n        f\"{name} {version}\"\n        for name, version in scrapy_components_versions()\n        if name != \"Scrapy\"\n    ]\n    logger.info(\"Versions: %(versions)s\", {\"versions\": \", \".join(versions)})\n\n\ndef log_reactor_info() -> None:\n    from twisted.internet import reactor\n\n    logger.debug(\"Using reactor: %s.%s\", reactor.__module__, reactor.__class__.__name__)\n    from twisted.internet import asyncioreactor\n\n    if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):\n        logger.debug(\n            \"Using asyncio event loop: %s.%s\",\n            reactor._asyncioEventloop.__module__,\n            reactor._asyncioEventloop.__class__.__name__,\n        )\n\n\nclass StreamLogger:\n    \"\"\"Fake file-like stream object that redirects writes to a logger instance\n\n    Taken from:\n        https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/\n    \"\"\"\n\n    def __init__(self, logger: logging.Logger, log_level: int = logging.INFO):\n        self.logger: logging.Logger = logger\n        self.log_level: int = log_level\n        self.linebuf: str = \"\"\n\n    def write(self, buf: str) -> None:\n        for line in buf.rstrip().splitlines():\n            self.logger.log(self.log_level, line.rstrip())\n\n    def flush(self) -> None:\n        for h in self.logger.handlers:\n            h.flush()\n\n\nclass LogCounterHandler(logging.Handler):\n    \"\"\"Record log levels count into a crawler stats\"\"\"\n\n    def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any):\n        super().__init__(*args, **kwargs)\n        self.crawler: Crawler = crawler\n\n    def emit(self, record: logging.LogRecord) -> None:\n        sname = f\"log_count/{record.levelname}\"\n        assert self.crawler.stats\n        self.crawler.stats.inc_value(sname)\n\n\ndef logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:\n    \"\"\"\n    Helper that takes the dictionary output from the methods in LogFormatter\n    and adapts it into a tuple of positional arguments for logger.log calls,\n    handling backward compatibility as well.\n    \"\"\"\n    if not {\"level\", \"msg\", \"args\"} <= set(logkws):\n        warnings.warn(\"Missing keys in LogFormatter method\", ScrapyDeprecationWarning)\n\n    if \"format\" in logkws:\n        warnings.warn(\n            \"`format` key in LogFormatter methods has been \"\n            \"deprecated, use `msg` instead\",\n            ScrapyDeprecationWarning,\n        )\n\n    level = logkws.get(\"level\", logging.INFO)\n    message = logkws.get(\"format\", logkws.get(\"msg\"))\n    # NOTE: This also handles 'args' being an empty dict, that case doesn't\n    # play well in logger.log calls\n    args = logkws if not logkws.get(\"args\") else logkws[\"args\"]\n\n    return (level, message, args)\n", "path": "scrapy/utils/log.py"}, {"content": "\"\"\"\nThis module provides some useful functions for working with\nscrapy.http.Response objects\n\"\"\"\nimport os\nimport re\nimport tempfile\nimport webbrowser\nfrom typing import Any, Callable, Iterable, Tuple, Union\nfrom weakref import WeakKeyDictionary\n\nfrom twisted.web import http\nfrom w3lib import html\n\nimport scrapy\nfrom scrapy.http.response import Response\nfrom scrapy.utils.decorators import deprecated\nfrom scrapy.utils.python import to_bytes, to_unicode\n\n_baseurl_cache: \"WeakKeyDictionary[Response, str]\" = WeakKeyDictionary()\n\n\ndef get_base_url(response: \"scrapy.http.response.text.TextResponse\") -> str:\n    \"\"\"Return the base url of the given response, joined with the response url\"\"\"\n    if response not in _baseurl_cache:\n        text = response.text[0:4096]\n        _baseurl_cache[response] = html.get_base_url(\n            text, response.url, response.encoding\n        )\n    return _baseurl_cache[response]\n\n\n_metaref_cache: \"WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]\" = (\n    WeakKeyDictionary()\n)\n\n\ndef get_meta_refresh(\n    response: \"scrapy.http.response.text.TextResponse\",\n    ignore_tags: Iterable[str] = (\"script\", \"noscript\"),\n) -> Union[Tuple[None, None], Tuple[float, str]]:\n    \"\"\"Parse the http-equiv refresh parameter from the given response\"\"\"\n    if response not in _metaref_cache:\n        text = response.text[0:4096]\n        _metaref_cache[response] = html.get_meta_refresh(\n            text, response.url, response.encoding, ignore_tags=ignore_tags\n        )\n    return _metaref_cache[response]\n\n\ndef response_status_message(status: Union[bytes, float, int, str]) -> str:\n    \"\"\"Return status code plus status text descriptive message\"\"\"\n    status_int = int(status)\n    message = http.RESPONSES.get(status_int, \"Unknown Status\")\n    return f\"{status_int} {to_unicode(message)}\"\n\n\n@deprecated\ndef response_httprepr(response: Response) -> bytes:\n    \"\"\"Return raw HTTP representation (as bytes) of the given response. This\n    is provided only for reference, since it's not the exact stream of bytes\n    that was received (that's not exposed by Twisted).\n    \"\"\"\n    values = [\n        b\"HTTP/1.1 \",\n        to_bytes(str(response.status)),\n        b\" \",\n        to_bytes(http.RESPONSES.get(response.status, b\"\")),\n        b\"\\r\\n\",\n    ]\n    if response.headers:\n        values.extend([response.headers.to_string(), b\"\\r\\n\"])\n    values.extend([b\"\\r\\n\", response.body])\n    return b\"\".join(values)\n\n\ndef open_in_browser(\n    response: Union[\n        \"scrapy.http.response.html.HtmlResponse\",\n        \"scrapy.http.response.text.TextResponse\",\n    ],\n    _openfunc: Callable[[str], Any] = webbrowser.open,\n) -> Any:\n    \"\"\"Open the given response in a local web browser, populating the <base>\n    tag for external links to work\n    \"\"\"\n    from scrapy.http import HtmlResponse, TextResponse\n\n    # XXX: this implementation is a bit dirty and could be improved\n    body = response.body\n    if isinstance(response, HtmlResponse):\n        if b\"<base\" not in body:\n            repl = rf'\\1<base href=\"{response.url}\">'\n            body = re.sub(b\"<!--.*?-->\", b\"\", body, flags=re.DOTALL)\n            body = re.sub(rb\"(<head(?:>|\\s.*?>))\", to_bytes(repl), body)\n        ext = \".html\"\n    elif isinstance(response, TextResponse):\n        ext = \".txt\"\n    else:\n        raise TypeError(\"Unsupported response type: \" f\"{response.__class__.__name__}\")\n    fd, fname = tempfile.mkstemp(ext)\n    os.write(fd, body)\n    os.close(fd)\n    return _openfunc(f\"file://{fname}\")\n", "path": "scrapy/utils/response.py"}], "after_files": [{"content": "from __future__ import annotations\n\nimport logging\nimport sys\nfrom logging.config import dictConfig\nfrom types import TracebackType\nfrom typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast\n\nfrom twisted.python import log as twisted_log\nfrom twisted.python.failure import Failure\n\nimport scrapy\nfrom scrapy.settings import Settings\nfrom scrapy.utils.versions import scrapy_components_versions\n\nif TYPE_CHECKING:\n    from scrapy.crawler import Crawler\n\nlogger = logging.getLogger(__name__)\n\n\ndef failure_to_exc_info(\n    failure: Failure,\n) -> Optional[Tuple[Type[BaseException], BaseException, Optional[TracebackType]]]:\n    \"\"\"Extract exc_info from Failure instances\"\"\"\n    if isinstance(failure, Failure):\n        assert failure.type\n        assert failure.value\n        return (\n            failure.type,\n            failure.value,\n            cast(Optional[TracebackType], failure.getTracebackObject()),\n        )\n    return None\n\n\nclass TopLevelFormatter(logging.Filter):\n    \"\"\"Keep only top level loggers's name (direct children from root) from\n    records.\n\n    This filter will replace Scrapy loggers' names with 'scrapy'. This mimics\n    the old Scrapy log behaviour and helps shortening long names.\n\n    Since it can't be set for just one logger (it won't propagate for its\n    children), it's going to be set in the root handler, with a parametrized\n    ``loggers`` list where it should act.\n    \"\"\"\n\n    def __init__(self, loggers: Optional[List[str]] = None):\n        self.loggers: List[str] = loggers or []\n\n    def filter(self, record: logging.LogRecord) -> bool:\n        if any(record.name.startswith(logger + \".\") for logger in self.loggers):\n            record.name = record.name.split(\".\", 1)[0]\n        return True\n\n\nDEFAULT_LOGGING = {\n    \"version\": 1,\n    \"disable_existing_loggers\": False,\n    \"loggers\": {\n        \"filelock\": {\n            \"level\": \"ERROR\",\n        },\n        \"hpack\": {\n            \"level\": \"ERROR\",\n        },\n        \"scrapy\": {\n            \"level\": \"DEBUG\",\n        },\n        \"twisted\": {\n            \"level\": \"ERROR\",\n        },\n    },\n}\n\n\ndef configure_logging(\n    settings: Union[Settings, dict, None] = None, install_root_handler: bool = True\n) -> None:\n    \"\"\"\n    Initialize logging defaults for Scrapy.\n\n    :param settings: settings used to create and configure a handler for the\n        root logger (default: None).\n    :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None``\n\n    :param install_root_handler: whether to install root logging handler\n        (default: True)\n    :type install_root_handler: bool\n\n    This function does:\n\n    - Route warnings and twisted logging through Python standard logging\n    - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively\n    - Route stdout to log if LOG_STDOUT setting is True\n\n    When ``install_root_handler`` is True (default), this function also\n    creates a handler for the root logger according to given settings\n    (see :ref:`topics-logging-settings`). You can override default options\n    using ``settings`` argument. When ``settings`` is empty or None, defaults\n    are used.\n    \"\"\"\n    if not sys.warnoptions:\n        # Route warnings through python logging\n        logging.captureWarnings(True)\n\n    observer = twisted_log.PythonLoggingObserver(\"twisted\")\n    observer.start()\n\n    dictConfig(DEFAULT_LOGGING)\n\n    if isinstance(settings, dict) or settings is None:\n        settings = Settings(settings)\n\n    if settings.getbool(\"LOG_STDOUT\"):\n        sys.stdout = StreamLogger(logging.getLogger(\"stdout\"))  # type: ignore[assignment]\n\n    if install_root_handler:\n        install_scrapy_root_handler(settings)\n\n\n_scrapy_root_handler: Optional[logging.Handler] = None\n\n\ndef install_scrapy_root_handler(settings: Settings) -> None:\n    global _scrapy_root_handler\n\n    if (\n        _scrapy_root_handler is not None\n        and _scrapy_root_handler in logging.root.handlers\n    ):\n        logging.root.removeHandler(_scrapy_root_handler)\n    logging.root.setLevel(logging.NOTSET)\n    _scrapy_root_handler = _get_handler(settings)\n    logging.root.addHandler(_scrapy_root_handler)\n\n\ndef get_scrapy_root_handler() -> Optional[logging.Handler]:\n    return _scrapy_root_handler\n\n\ndef _get_handler(settings: Settings) -> logging.Handler:\n    \"\"\"Return a log handler object according to settings\"\"\"\n    filename = settings.get(\"LOG_FILE\")\n    handler: logging.Handler\n    if filename:\n        mode = \"a\" if settings.getbool(\"LOG_FILE_APPEND\") else \"w\"\n        encoding = settings.get(\"LOG_ENCODING\")\n        handler = logging.FileHandler(filename, mode=mode, encoding=encoding)\n    elif settings.getbool(\"LOG_ENABLED\"):\n        handler = logging.StreamHandler()\n    else:\n        handler = logging.NullHandler()\n\n    formatter = logging.Formatter(\n        fmt=settings.get(\"LOG_FORMAT\"), datefmt=settings.get(\"LOG_DATEFORMAT\")\n    )\n    handler.setFormatter(formatter)\n    handler.setLevel(settings.get(\"LOG_LEVEL\"))\n    if settings.getbool(\"LOG_SHORT_NAMES\"):\n        handler.addFilter(TopLevelFormatter([\"scrapy\"]))\n    return handler\n\n\ndef log_scrapy_info(settings: Settings) -> None:\n    logger.info(\n        \"Scrapy %(version)s started (bot: %(bot)s)\",\n        {\"version\": scrapy.__version__, \"bot\": settings[\"BOT_NAME\"]},\n    )\n    versions = [\n        f\"{name} {version}\"\n        for name, version in scrapy_components_versions()\n        if name != \"Scrapy\"\n    ]\n    logger.info(\"Versions: %(versions)s\", {\"versions\": \", \".join(versions)})\n\n\ndef log_reactor_info() -> None:\n    from twisted.internet import reactor\n\n    logger.debug(\"Using reactor: %s.%s\", reactor.__module__, reactor.__class__.__name__)\n    from twisted.internet import asyncioreactor\n\n    if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):\n        logger.debug(\n            \"Using asyncio event loop: %s.%s\",\n            reactor._asyncioEventloop.__module__,\n            reactor._asyncioEventloop.__class__.__name__,\n        )\n\n\nclass StreamLogger:\n    \"\"\"Fake file-like stream object that redirects writes to a logger instance\n\n    Taken from:\n        https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/\n    \"\"\"\n\n    def __init__(self, logger: logging.Logger, log_level: int = logging.INFO):\n        self.logger: logging.Logger = logger\n        self.log_level: int = log_level\n        self.linebuf: str = \"\"\n\n    def write(self, buf: str) -> None:\n        for line in buf.rstrip().splitlines():\n            self.logger.log(self.log_level, line.rstrip())\n\n    def flush(self) -> None:\n        for h in self.logger.handlers:\n            h.flush()\n\n\nclass LogCounterHandler(logging.Handler):\n    \"\"\"Record log levels count into a crawler stats\"\"\"\n\n    def __init__(self, crawler: Crawler, *args: Any, **kwargs: Any):\n        super().__init__(*args, **kwargs)\n        self.crawler: Crawler = crawler\n\n    def emit(self, record: logging.LogRecord) -> None:\n        sname = f\"log_count/{record.levelname}\"\n        assert self.crawler.stats\n        self.crawler.stats.inc_value(sname)\n\n\ndef logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:\n    \"\"\"\n    Helper that takes the dictionary output from the methods in LogFormatter\n    and adapts it into a tuple of positional arguments for logger.log calls,\n    handling backward compatibility as well.\n    \"\"\"\n\n    level = logkws.get(\"level\", logging.INFO)\n    message = logkws.get(\"msg\") or \"\"\n    # NOTE: This also handles 'args' being an empty dict, that case doesn't\n    # play well in logger.log calls\n    args = logkws if not logkws.get(\"args\") else logkws[\"args\"]\n\n    return (level, message, args)\n", "path": "scrapy/utils/log.py"}, {"content": "\"\"\"\nThis module provides some useful functions for working with\nscrapy.http.Response objects\n\"\"\"\nimport os\nimport re\nimport tempfile\nimport webbrowser\nfrom typing import Any, Callable, Iterable, Tuple, Union\nfrom weakref import WeakKeyDictionary\n\nfrom twisted.web import http\nfrom w3lib import html\n\nimport scrapy\nfrom scrapy.http.response import Response\nfrom scrapy.utils.python import to_bytes, to_unicode\n\n_baseurl_cache: \"WeakKeyDictionary[Response, str]\" = WeakKeyDictionary()\n\n\ndef get_base_url(response: \"scrapy.http.response.text.TextResponse\") -> str:\n    \"\"\"Return the base url of the given response, joined with the response url\"\"\"\n    if response not in _baseurl_cache:\n        text = response.text[0:4096]\n        _baseurl_cache[response] = html.get_base_url(\n            text, response.url, response.encoding\n        )\n    return _baseurl_cache[response]\n\n\n_metaref_cache: \"WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]\" = (\n    WeakKeyDictionary()\n)\n\n\ndef get_meta_refresh(\n    response: \"scrapy.http.response.text.TextResponse\",\n    ignore_tags: Iterable[str] = (\"script\", \"noscript\"),\n) -> Union[Tuple[None, None], Tuple[float, str]]:\n    \"\"\"Parse the http-equiv refresh parameter from the given response\"\"\"\n    if response not in _metaref_cache:\n        text = response.text[0:4096]\n        _metaref_cache[response] = html.get_meta_refresh(\n            text, response.url, response.encoding, ignore_tags=ignore_tags\n        )\n    return _metaref_cache[response]\n\n\ndef response_status_message(status: Union[bytes, float, int, str]) -> str:\n    \"\"\"Return status code plus status text descriptive message\"\"\"\n    status_int = int(status)\n    message = http.RESPONSES.get(status_int, \"Unknown Status\")\n    return f\"{status_int} {to_unicode(message)}\"\n\n\ndef open_in_browser(\n    response: Union[\n        \"scrapy.http.response.html.HtmlResponse\",\n        \"scrapy.http.response.text.TextResponse\",\n    ],\n    _openfunc: Callable[[str], Any] = webbrowser.open,\n) -> Any:\n    \"\"\"Open the given response in a local web browser, populating the <base>\n    tag for external links to work\n    \"\"\"\n    from scrapy.http import HtmlResponse, TextResponse\n\n    # XXX: this implementation is a bit dirty and could be improved\n    body = response.body\n    if isinstance(response, HtmlResponse):\n        if b\"<base\" not in body:\n            repl = rf'\\1<base href=\"{response.url}\">'\n            body = re.sub(b\"<!--.*?-->\", b\"\", body, flags=re.DOTALL)\n            body = re.sub(rb\"(<head(?:>|\\s.*?>))\", to_bytes(repl), body)\n        ext = \".html\"\n    elif isinstance(response, TextResponse):\n        ext = \".txt\"\n    else:\n        raise TypeError(\"Unsupported response type: \" f\"{response.__class__.__name__}\")\n    fd, fname = tempfile.mkstemp(ext)\n    os.write(fd, body)\n    os.close(fd)\n    return _openfunc(f\"file://{fname}\")\n", "path": "scrapy/utils/response.py"}]}