We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Bug in expected_http_body_size
##### Steps to reproduce the problem:

1. Any response with more than one content-length header


##### Any other comments? What have you tried so far?

headers is MultiDict so this:
size = int(headers["content-length"])
can be change to:
size = int(headers.get_all("content-length")[0])

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `mitmproxy/net/http/http1/read.py`
Content:
```
1 import time
2 import sys
3 import re
4 
5 from mitmproxy.net.http import request
6 from mitmproxy.net.http import response
7 from mitmproxy.net.http import headers
8 from mitmproxy.net.http import url
9 from mitmproxy.net import check
10 from mitmproxy import exceptions
11 
12 
13 def get_header_tokens(headers, key):
14     """
15         Retrieve all tokens for a header key. A number of different headers
16         follow a pattern where each header line can containe comma-separated
17         tokens, and headers can be set multiple times.
18     """
19     if key not in headers:
20         return []
21     tokens = headers[key].split(",")
22     return [token.strip() for token in tokens]
23 
24 
25 def read_request(rfile, body_size_limit=None):
26     request = read_request_head(rfile)
27     expected_body_size = expected_http_body_size(request)
28     request.data.content = b"".join(read_body(rfile, expected_body_size, limit=body_size_limit))
29     request.timestamp_end = time.time()
30     return request
31 
32 
33 def read_request_head(rfile):
34     """
35     Parse an HTTP request head (request line + headers) from an input stream
36 
37     Args:
38         rfile: The input stream
39 
40     Returns:
41         The HTTP request object (without body)
42 
43     Raises:
44         exceptions.HttpReadDisconnect: No bytes can be read from rfile.
45         exceptions.HttpSyntaxException: The input is malformed HTTP.
46         exceptions.HttpException: Any other error occured.
47     """
48     timestamp_start = time.time()
49     if hasattr(rfile, "reset_timestamps"):
50         rfile.reset_timestamps()
51 
52     form, method, scheme, host, port, path, http_version = _read_request_line(rfile)
53     headers = _read_headers(rfile)
54 
55     if hasattr(rfile, "first_byte_timestamp"):
56         # more accurate timestamp_start
57         timestamp_start = rfile.first_byte_timestamp
58 
59     return request.Request(
60         form, method, scheme, host, port, path, http_version, headers, None, timestamp_start
61     )
62 
63 
64 def read_response(rfile, request, body_size_limit=None):
65     response = read_response_head(rfile)
66     expected_body_size = expected_http_body_size(request, response)
67     response.data.content = b"".join(read_body(rfile, expected_body_size, body_size_limit))
68     response.timestamp_end = time.time()
69     return response
70 
71 
72 def read_response_head(rfile):
73     """
74     Parse an HTTP response head (response line + headers) from an input stream
75 
76     Args:
77         rfile: The input stream
78 
79     Returns:
80         The HTTP request object (without body)
81 
82     Raises:
83         exceptions.HttpReadDisconnect: No bytes can be read from rfile.
84         exceptions.HttpSyntaxException: The input is malformed HTTP.
85         exceptions.HttpException: Any other error occured.
86     """
87 
88     timestamp_start = time.time()
89     if hasattr(rfile, "reset_timestamps"):
90         rfile.reset_timestamps()
91 
92     http_version, status_code, message = _read_response_line(rfile)
93     headers = _read_headers(rfile)
94 
95     if hasattr(rfile, "first_byte_timestamp"):
96         # more accurate timestamp_start
97         timestamp_start = rfile.first_byte_timestamp
98 
99     return response.Response(http_version, status_code, message, headers, None, timestamp_start)
100 
101 
102 def read_body(rfile, expected_size, limit=None, max_chunk_size=4096):
103     """
104         Read an HTTP message body
105 
106         Args:
107             rfile: The input stream
108             expected_size: The expected body size (see :py:meth:`expected_body_size`)
109             limit: Maximum body size
110             max_chunk_size: Maximium chunk size that gets yielded
111 
112         Returns:
113             A generator that yields byte chunks of the content.
114 
115         Raises:
116             exceptions.HttpException, if an error occurs
117 
118         Caveats:
119             max_chunk_size is not considered if the transfer encoding is chunked.
120     """
121     if not limit or limit < 0:
122         limit = sys.maxsize
123     if not max_chunk_size:
124         max_chunk_size = limit
125 
126     if expected_size is None:
127         for x in _read_chunked(rfile, limit):
128             yield x
129     elif expected_size >= 0:
130         if limit is not None and expected_size > limit:
131             raise exceptions.HttpException(
132                 "HTTP Body too large. "
133                 "Limit is {}, content length was advertised as {}".format(limit, expected_size)
134             )
135         bytes_left = expected_size
136         while bytes_left:
137             chunk_size = min(bytes_left, max_chunk_size)
138             content = rfile.read(chunk_size)
139             if len(content) < chunk_size:
140                 raise exceptions.HttpException("Unexpected EOF")
141             yield content
142             bytes_left -= chunk_size
143     else:
144         bytes_left = limit
145         while bytes_left:
146             chunk_size = min(bytes_left, max_chunk_size)
147             content = rfile.read(chunk_size)
148             if not content:
149                 return
150             yield content
151             bytes_left -= chunk_size
152         not_done = rfile.read(1)
153         if not_done:
154             raise exceptions.HttpException("HTTP body too large. Limit is {}.".format(limit))
155 
156 
157 def connection_close(http_version, headers):
158     """
159         Checks the message to see if the client connection should be closed
160         according to RFC 2616 Section 8.1.
161         If we don't have a Connection header, HTTP 1.1 connections are assumed
162         to be persistent.
163     """
164     if "connection" in headers:
165         tokens = get_header_tokens(headers, "connection")
166         if "close" in tokens:
167             return True
168         elif "keep-alive" in tokens:
169             return False
170 
171     return http_version != "HTTP/1.1" and http_version != b"HTTP/1.1"
172 
173 
174 def expected_http_body_size(request, response=None):
175     """
176         Returns:
177             The expected body length:
178             - a positive integer, if the size is known in advance
179             - None, if the size in unknown in advance (chunked encoding)
180             - -1, if all data should be read until end of stream.
181 
182         Raises:
183             exceptions.HttpSyntaxException, if the content length header is invalid
184     """
185     # Determine response size according to
186     # http://tools.ietf.org/html/rfc7230#section-3.3
187     if not response:
188         headers = request.headers
189         response_code = None
190         is_request = True
191     else:
192         headers = response.headers
193         response_code = response.status_code
194         is_request = False
195 
196     if is_request:
197         if headers.get("expect", "").lower() == "100-continue":
198             return 0
199     else:
200         if request.method.upper() == "HEAD":
201             return 0
202         if 100 <= response_code <= 199:
203             return 0
204         if response_code == 200 and request.method.upper() == "CONNECT":
205             return 0
206         if response_code in (204, 304):
207             return 0
208 
209     if "chunked" in headers.get("transfer-encoding", "").lower():
210         return None
211     if "content-length" in headers:
212         try:
213             size = int(headers["content-length"])
214             if size < 0:
215                 raise ValueError()
216             return size
217         except ValueError:
218             raise exceptions.HttpSyntaxException("Unparseable Content Length")
219     if is_request:
220         return 0
221     return -1
222 
223 
224 def _get_first_line(rfile):
225     try:
226         line = rfile.readline()
227         if line == b"\r\n" or line == b"\n":
228             # Possible leftover from previous message
229             line = rfile.readline()
230     except (exceptions.TcpDisconnect, exceptions.TlsException):
231         raise exceptions.HttpReadDisconnect("Remote disconnected")
232     if not line:
233         raise exceptions.HttpReadDisconnect("Remote disconnected")
234     return line.strip()
235 
236 
237 def _read_request_line(rfile):
238     try:
239         line = _get_first_line(rfile)
240     except exceptions.HttpReadDisconnect:
241         # We want to provide a better error message.
242         raise exceptions.HttpReadDisconnect("Client disconnected")
243 
244     try:
245         method, path, http_version = line.split()
246 
247         if path == b"*" or path.startswith(b"/"):
248             form = "relative"
249             scheme, host, port = None, None, None
250         elif method == b"CONNECT":
251             form = "authority"
252             host, port = _parse_authority_form(path)
253             scheme, path = None, None
254         else:
255             form = "absolute"
256             scheme, host, port, path = url.parse(path)
257 
258         _check_http_version(http_version)
259     except ValueError:
260         raise exceptions.HttpSyntaxException("Bad HTTP request line: {}".format(line))
261 
262     return form, method, scheme, host, port, path, http_version
263 
264 
265 def _parse_authority_form(hostport):
266     """
267         Returns (host, port) if hostport is a valid authority-form host specification.
268         http://tools.ietf.org/html/draft-luotonen-web-proxy-tunneling-01 section 3.1
269 
270         Raises:
271             ValueError, if the input is malformed
272     """
273     try:
274         host, port = hostport.rsplit(b":", 1)
275         if host.startswith(b"[") and host.endswith(b"]"):
276             host = host[1:-1]
277         port = int(port)
278         if not check.is_valid_host(host) or not check.is_valid_port(port):
279             raise ValueError()
280     except ValueError:
281         raise exceptions.HttpSyntaxException("Invalid host specification: {}".format(hostport))
282 
283     return host, port
284 
285 
286 def _read_response_line(rfile):
287     try:
288         line = _get_first_line(rfile)
289     except exceptions.HttpReadDisconnect:
290         # We want to provide a better error message.
291         raise exceptions.HttpReadDisconnect("Server disconnected")
292 
293     try:
294         parts = line.split(None, 2)
295         if len(parts) == 2:  # handle missing message gracefully
296             parts.append(b"")
297 
298         http_version, status_code, message = parts
299         status_code = int(status_code)
300         _check_http_version(http_version)
301 
302     except ValueError:
303         raise exceptions.HttpSyntaxException("Bad HTTP response line: {}".format(line))
304 
305     return http_version, status_code, message
306 
307 
308 def _check_http_version(http_version):
309     if not re.match(br"^HTTP/\d\.\d$", http_version):
310         raise exceptions.HttpSyntaxException("Unknown HTTP version: {}".format(http_version))
311 
312 
313 def _read_headers(rfile):
314     """
315         Read a set of headers.
316         Stop once a blank line is reached.
317 
318         Returns:
319             A headers object
320 
321         Raises:
322             exceptions.HttpSyntaxException
323     """
324     ret = []
325     while True:
326         line = rfile.readline()
327         if not line or line == b"\r\n" or line == b"\n":
328             break
329         if line[0] in b" \t":
330             if not ret:
331                 raise exceptions.HttpSyntaxException("Invalid headers")
332             # continued header
333             ret[-1] = (ret[-1][0], ret[-1][1] + b'\r\n ' + line.strip())
334         else:
335             try:
336                 name, value = line.split(b":", 1)
337                 value = value.strip()
338                 if not name:
339                     raise ValueError()
340                 ret.append((name, value))
341             except ValueError:
342                 raise exceptions.HttpSyntaxException(
343                     "Invalid header line: %s" % repr(line)
344                 )
345     return headers.Headers(ret)
346 
347 
348 def _read_chunked(rfile, limit=sys.maxsize):
349     """
350     Read a HTTP body with chunked transfer encoding.
351 
352     Args:
353         rfile: the input file
354         limit: A positive integer
355     """
356     total = 0
357     while True:
358         line = rfile.readline(128)
359         if line == b"":
360             raise exceptions.HttpException("Connection closed prematurely")
361         if line != b"\r\n" and line != b"\n":
362             try:
363                 length = int(line, 16)
364             except ValueError:
365                 raise exceptions.HttpSyntaxException("Invalid chunked encoding length: {}".format(line))
366             total += length
367             if total > limit:
368                 raise exceptions.HttpException(
369                     "HTTP Body too large. Limit is {}, "
370                     "chunked content longer than {}".format(limit, total)
371                 )
372             chunk = rfile.read(length)
373             suffix = rfile.readline(5)
374             if suffix != b"\r\n":
375                 raise exceptions.HttpSyntaxException("Malformed chunked body")
376             if length == 0:
377                 return
378             yield chunk
379
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py
--- a/mitmproxy/net/http/http1/read.py
+++ b/mitmproxy/net/http/http1/read.py
@@ -210,7 +210,11 @@
         return None
     if "content-length" in headers:
         try:
-            size = int(headers["content-length"])
+            sizes = headers.get_all("content-length")
+            different_content_length_headers = any(x != sizes[0] for x in sizes)
+            if different_content_length_headers:
+                raise exceptions.HttpSyntaxException("Conflicting Content Length Headers")
+            size = int(sizes[0])
             if size < 0:
                 raise ValueError()
             return size

{"golden_diff": "diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py\n--- a/mitmproxy/net/http/http1/read.py\n+++ b/mitmproxy/net/http/http1/read.py\n@@ -210,7 +210,11 @@\n         return None\n     if \"content-length\" in headers:\n         try:\n-            size = int(headers[\"content-length\"])\n+            sizes = headers.get_all(\"content-length\")\n+            different_content_length_headers = any(x != sizes[0] for x in sizes)\n+            if different_content_length_headers:\n+                raise exceptions.HttpSyntaxException(\"Conflicting Content Length Headers\")\n+            size = int(sizes[0])\n             if size < 0:\n                 raise ValueError()\n             return size\n", "issue": "Bug in expected_http_body_size\n##### Steps to reproduce the problem:\r\n\r\n1. Any response with more than one content-length header\r\n\r\n\r\n##### Any other comments? What have you tried so far?\r\n\r\nheaders is MultiDict so this:\r\nsize = int(headers[\"content-length\"])\r\ncan be change to:\r\nsize = int(headers.get_all(\"content-length\")[0])\n", "before_files": [{"content": "import time\nimport sys\nimport re\n\nfrom mitmproxy.net.http import request\nfrom mitmproxy.net.http import response\nfrom mitmproxy.net.http import headers\nfrom mitmproxy.net.http import url\nfrom mitmproxy.net import check\nfrom mitmproxy import exceptions\n\n\ndef get_header_tokens(headers, key):\n    \"\"\"\n        Retrieve all tokens for a header key. A number of different headers\n        follow a pattern where each header line can containe comma-separated\n        tokens, and headers can be set multiple times.\n    \"\"\"\n    if key not in headers:\n        return []\n    tokens = headers[key].split(\",\")\n    return [token.strip() for token in tokens]\n\n\ndef read_request(rfile, body_size_limit=None):\n    request = read_request_head(rfile)\n    expected_body_size = expected_http_body_size(request)\n    request.data.content = b\"\".join(read_body(rfile, expected_body_size, limit=body_size_limit))\n    request.timestamp_end = time.time()\n    return request\n\n\ndef read_request_head(rfile):\n    \"\"\"\n    Parse an HTTP request head (request line + headers) from an input stream\n\n    Args:\n        rfile: The input stream\n\n    Returns:\n        The HTTP request object (without body)\n\n    Raises:\n        exceptions.HttpReadDisconnect: No bytes can be read from rfile.\n        exceptions.HttpSyntaxException: The input is malformed HTTP.\n        exceptions.HttpException: Any other error occured.\n    \"\"\"\n    timestamp_start = time.time()\n    if hasattr(rfile, \"reset_timestamps\"):\n        rfile.reset_timestamps()\n\n    form, method, scheme, host, port, path, http_version = _read_request_line(rfile)\n    headers = _read_headers(rfile)\n\n    if hasattr(rfile, \"first_byte_timestamp\"):\n        # more accurate timestamp_start\n        timestamp_start = rfile.first_byte_timestamp\n\n    return request.Request(\n        form, method, scheme, host, port, path, http_version, headers, None, timestamp_start\n    )\n\n\ndef read_response(rfile, request, body_size_limit=None):\n    response = read_response_head(rfile)\n    expected_body_size = expected_http_body_size(request, response)\n    response.data.content = b\"\".join(read_body(rfile, expected_body_size, body_size_limit))\n    response.timestamp_end = time.time()\n    return response\n\n\ndef read_response_head(rfile):\n    \"\"\"\n    Parse an HTTP response head (response line + headers) from an input stream\n\n    Args:\n        rfile: The input stream\n\n    Returns:\n        The HTTP request object (without body)\n\n    Raises:\n        exceptions.HttpReadDisconnect: No bytes can be read from rfile.\n        exceptions.HttpSyntaxException: The input is malformed HTTP.\n        exceptions.HttpException: Any other error occured.\n    \"\"\"\n\n    timestamp_start = time.time()\n    if hasattr(rfile, \"reset_timestamps\"):\n        rfile.reset_timestamps()\n\n    http_version, status_code, message = _read_response_line(rfile)\n    headers = _read_headers(rfile)\n\n    if hasattr(rfile, \"first_byte_timestamp\"):\n        # more accurate timestamp_start\n        timestamp_start = rfile.first_byte_timestamp\n\n    return response.Response(http_version, status_code, message, headers, None, timestamp_start)\n\n\ndef read_body(rfile, expected_size, limit=None, max_chunk_size=4096):\n    \"\"\"\n        Read an HTTP message body\n\n        Args:\n            rfile: The input stream\n            expected_size: The expected body size (see :py:meth:`expected_body_size`)\n            limit: Maximum body size\n            max_chunk_size: Maximium chunk size that gets yielded\n\n        Returns:\n            A generator that yields byte chunks of the content.\n\n        Raises:\n            exceptions.HttpException, if an error occurs\n\n        Caveats:\n            max_chunk_size is not considered if the transfer encoding is chunked.\n    \"\"\"\n    if not limit or limit < 0:\n        limit = sys.maxsize\n    if not max_chunk_size:\n        max_chunk_size = limit\n\n    if expected_size is None:\n        for x in _read_chunked(rfile, limit):\n            yield x\n    elif expected_size >= 0:\n        if limit is not None and expected_size > limit:\n            raise exceptions.HttpException(\n                \"HTTP Body too large. \"\n                \"Limit is {}, content length was advertised as {}\".format(limit, expected_size)\n            )\n        bytes_left = expected_size\n        while bytes_left:\n            chunk_size = min(bytes_left, max_chunk_size)\n            content = rfile.read(chunk_size)\n            if len(content) < chunk_size:\n                raise exceptions.HttpException(\"Unexpected EOF\")\n            yield content\n            bytes_left -= chunk_size\n    else:\n        bytes_left = limit\n        while bytes_left:\n            chunk_size = min(bytes_left, max_chunk_size)\n            content = rfile.read(chunk_size)\n            if not content:\n                return\n            yield content\n            bytes_left -= chunk_size\n        not_done = rfile.read(1)\n        if not_done:\n            raise exceptions.HttpException(\"HTTP body too large. Limit is {}.\".format(limit))\n\n\ndef connection_close(http_version, headers):\n    \"\"\"\n        Checks the message to see if the client connection should be closed\n        according to RFC 2616 Section 8.1.\n        If we don't have a Connection header, HTTP 1.1 connections are assumed\n        to be persistent.\n    \"\"\"\n    if \"connection\" in headers:\n        tokens = get_header_tokens(headers, \"connection\")\n        if \"close\" in tokens:\n            return True\n        elif \"keep-alive\" in tokens:\n            return False\n\n    return http_version != \"HTTP/1.1\" and http_version != b\"HTTP/1.1\"\n\n\ndef expected_http_body_size(request, response=None):\n    \"\"\"\n        Returns:\n            The expected body length:\n            - a positive integer, if the size is known in advance\n            - None, if the size in unknown in advance (chunked encoding)\n            - -1, if all data should be read until end of stream.\n\n        Raises:\n            exceptions.HttpSyntaxException, if the content length header is invalid\n    \"\"\"\n    # Determine response size according to\n    # http://tools.ietf.org/html/rfc7230#section-3.3\n    if not response:\n        headers = request.headers\n        response_code = None\n        is_request = True\n    else:\n        headers = response.headers\n        response_code = response.status_code\n        is_request = False\n\n    if is_request:\n        if headers.get(\"expect\", \"\").lower() == \"100-continue\":\n            return 0\n    else:\n        if request.method.upper() == \"HEAD\":\n            return 0\n        if 100 <= response_code <= 199:\n            return 0\n        if response_code == 200 and request.method.upper() == \"CONNECT\":\n            return 0\n        if response_code in (204, 304):\n            return 0\n\n    if \"chunked\" in headers.get(\"transfer-encoding\", \"\").lower():\n        return None\n    if \"content-length\" in headers:\n        try:\n            size = int(headers[\"content-length\"])\n            if size < 0:\n                raise ValueError()\n            return size\n        except ValueError:\n            raise exceptions.HttpSyntaxException(\"Unparseable Content Length\")\n    if is_request:\n        return 0\n    return -1\n\n\ndef _get_first_line(rfile):\n    try:\n        line = rfile.readline()\n        if line == b\"\\r\\n\" or line == b\"\\n\":\n            # Possible leftover from previous message\n            line = rfile.readline()\n    except (exceptions.TcpDisconnect, exceptions.TlsException):\n        raise exceptions.HttpReadDisconnect(\"Remote disconnected\")\n    if not line:\n        raise exceptions.HttpReadDisconnect(\"Remote disconnected\")\n    return line.strip()\n\n\ndef _read_request_line(rfile):\n    try:\n        line = _get_first_line(rfile)\n    except exceptions.HttpReadDisconnect:\n        # We want to provide a better error message.\n        raise exceptions.HttpReadDisconnect(\"Client disconnected\")\n\n    try:\n        method, path, http_version = line.split()\n\n        if path == b\"*\" or path.startswith(b\"/\"):\n            form = \"relative\"\n            scheme, host, port = None, None, None\n        elif method == b\"CONNECT\":\n            form = \"authority\"\n            host, port = _parse_authority_form(path)\n            scheme, path = None, None\n        else:\n            form = \"absolute\"\n            scheme, host, port, path = url.parse(path)\n\n        _check_http_version(http_version)\n    except ValueError:\n        raise exceptions.HttpSyntaxException(\"Bad HTTP request line: {}\".format(line))\n\n    return form, method, scheme, host, port, path, http_version\n\n\ndef _parse_authority_form(hostport):\n    \"\"\"\n        Returns (host, port) if hostport is a valid authority-form host specification.\n        http://tools.ietf.org/html/draft-luotonen-web-proxy-tunneling-01 section 3.1\n\n        Raises:\n            ValueError, if the input is malformed\n    \"\"\"\n    try:\n        host, port = hostport.rsplit(b\":\", 1)\n        if host.startswith(b\"[\") and host.endswith(b\"]\"):\n            host = host[1:-1]\n        port = int(port)\n        if not check.is_valid_host(host) or not check.is_valid_port(port):\n            raise ValueError()\n    except ValueError:\n        raise exceptions.HttpSyntaxException(\"Invalid host specification: {}\".format(hostport))\n\n    return host, port\n\n\ndef _read_response_line(rfile):\n    try:\n        line = _get_first_line(rfile)\n    except exceptions.HttpReadDisconnect:\n        # We want to provide a better error message.\n        raise exceptions.HttpReadDisconnect(\"Server disconnected\")\n\n    try:\n        parts = line.split(None, 2)\n        if len(parts) == 2:  # handle missing message gracefully\n            parts.append(b\"\")\n\n        http_version, status_code, message = parts\n        status_code = int(status_code)\n        _check_http_version(http_version)\n\n    except ValueError:\n        raise exceptions.HttpSyntaxException(\"Bad HTTP response line: {}\".format(line))\n\n    return http_version, status_code, message\n\n\ndef _check_http_version(http_version):\n    if not re.match(br\"^HTTP/\\d\\.\\d$\", http_version):\n        raise exceptions.HttpSyntaxException(\"Unknown HTTP version: {}\".format(http_version))\n\n\ndef _read_headers(rfile):\n    \"\"\"\n        Read a set of headers.\n        Stop once a blank line is reached.\n\n        Returns:\n            A headers object\n\n        Raises:\n            exceptions.HttpSyntaxException\n    \"\"\"\n    ret = []\n    while True:\n        line = rfile.readline()\n        if not line or line == b\"\\r\\n\" or line == b\"\\n\":\n            break\n        if line[0] in b\" \\t\":\n            if not ret:\n                raise exceptions.HttpSyntaxException(\"Invalid headers\")\n            # continued header\n            ret[-1] = (ret[-1][0], ret[-1][1] + b'\\r\\n ' + line.strip())\n        else:\n            try:\n                name, value = line.split(b\":\", 1)\n                value = value.strip()\n                if not name:\n                    raise ValueError()\n                ret.append((name, value))\n            except ValueError:\n                raise exceptions.HttpSyntaxException(\n                    \"Invalid header line: %s\" % repr(line)\n                )\n    return headers.Headers(ret)\n\n\ndef _read_chunked(rfile, limit=sys.maxsize):\n    \"\"\"\n    Read a HTTP body with chunked transfer encoding.\n\n    Args:\n        rfile: the input file\n        limit: A positive integer\n    \"\"\"\n    total = 0\n    while True:\n        line = rfile.readline(128)\n        if line == b\"\":\n            raise exceptions.HttpException(\"Connection closed prematurely\")\n        if line != b\"\\r\\n\" and line != b\"\\n\":\n            try:\n                length = int(line, 16)\n            except ValueError:\n                raise exceptions.HttpSyntaxException(\"Invalid chunked encoding length: {}\".format(line))\n            total += length\n            if total > limit:\n                raise exceptions.HttpException(\n                    \"HTTP Body too large. Limit is {}, \"\n                    \"chunked content longer than {}\".format(limit, total)\n                )\n            chunk = rfile.read(length)\n            suffix = rfile.readline(5)\n            if suffix != b\"\\r\\n\":\n                raise exceptions.HttpSyntaxException(\"Malformed chunked body\")\n            if length == 0:\n                return\n            yield chunk\n", "path": "mitmproxy/net/http/http1/read.py"}], "after_files": [{"content": "import time\nimport sys\nimport re\n\nfrom mitmproxy.net.http import request\nfrom mitmproxy.net.http import response\nfrom mitmproxy.net.http import headers\nfrom mitmproxy.net.http import url\nfrom mitmproxy.net import check\nfrom mitmproxy import exceptions\n\n\ndef get_header_tokens(headers, key):\n    \"\"\"\n        Retrieve all tokens for a header key. A number of different headers\n        follow a pattern where each header line can containe comma-separated\n        tokens, and headers can be set multiple times.\n    \"\"\"\n    if key not in headers:\n        return []\n    tokens = headers[key].split(\",\")\n    return [token.strip() for token in tokens]\n\n\ndef read_request(rfile, body_size_limit=None):\n    request = read_request_head(rfile)\n    expected_body_size = expected_http_body_size(request)\n    request.data.content = b\"\".join(read_body(rfile, expected_body_size, limit=body_size_limit))\n    request.timestamp_end = time.time()\n    return request\n\n\ndef read_request_head(rfile):\n    \"\"\"\n    Parse an HTTP request head (request line + headers) from an input stream\n\n    Args:\n        rfile: The input stream\n\n    Returns:\n        The HTTP request object (without body)\n\n    Raises:\n        exceptions.HttpReadDisconnect: No bytes can be read from rfile.\n        exceptions.HttpSyntaxException: The input is malformed HTTP.\n        exceptions.HttpException: Any other error occured.\n    \"\"\"\n    timestamp_start = time.time()\n    if hasattr(rfile, \"reset_timestamps\"):\n        rfile.reset_timestamps()\n\n    form, method, scheme, host, port, path, http_version = _read_request_line(rfile)\n    headers = _read_headers(rfile)\n\n    if hasattr(rfile, \"first_byte_timestamp\"):\n        # more accurate timestamp_start\n        timestamp_start = rfile.first_byte_timestamp\n\n    return request.Request(\n        form, method, scheme, host, port, path, http_version, headers, None, timestamp_start\n    )\n\n\ndef read_response(rfile, request, body_size_limit=None):\n    response = read_response_head(rfile)\n    expected_body_size = expected_http_body_size(request, response)\n    response.data.content = b\"\".join(read_body(rfile, expected_body_size, body_size_limit))\n    response.timestamp_end = time.time()\n    return response\n\n\ndef read_response_head(rfile):\n    \"\"\"\n    Parse an HTTP response head (response line + headers) from an input stream\n\n    Args:\n        rfile: The input stream\n\n    Returns:\n        The HTTP request object (without body)\n\n    Raises:\n        exceptions.HttpReadDisconnect: No bytes can be read from rfile.\n        exceptions.HttpSyntaxException: The input is malformed HTTP.\n        exceptions.HttpException: Any other error occured.\n    \"\"\"\n\n    timestamp_start = time.time()\n    if hasattr(rfile, \"reset_timestamps\"):\n        rfile.reset_timestamps()\n\n    http_version, status_code, message = _read_response_line(rfile)\n    headers = _read_headers(rfile)\n\n    if hasattr(rfile, \"first_byte_timestamp\"):\n        # more accurate timestamp_start\n        timestamp_start = rfile.first_byte_timestamp\n\n    return response.Response(http_version, status_code, message, headers, None, timestamp_start)\n\n\ndef read_body(rfile, expected_size, limit=None, max_chunk_size=4096):\n    \"\"\"\n        Read an HTTP message body\n\n        Args:\n            rfile: The input stream\n            expected_size: The expected body size (see :py:meth:`expected_body_size`)\n            limit: Maximum body size\n            max_chunk_size: Maximium chunk size that gets yielded\n\n        Returns:\n            A generator that yields byte chunks of the content.\n\n        Raises:\n            exceptions.HttpException, if an error occurs\n\n        Caveats:\n            max_chunk_size is not considered if the transfer encoding is chunked.\n    \"\"\"\n    if not limit or limit < 0:\n        limit = sys.maxsize\n    if not max_chunk_size:\n        max_chunk_size = limit\n\n    if expected_size is None:\n        for x in _read_chunked(rfile, limit):\n            yield x\n    elif expected_size >= 0:\n        if limit is not None and expected_size > limit:\n            raise exceptions.HttpException(\n                \"HTTP Body too large. \"\n                \"Limit is {}, content length was advertised as {}\".format(limit, expected_size)\n            )\n        bytes_left = expected_size\n        while bytes_left:\n            chunk_size = min(bytes_left, max_chunk_size)\n            content = rfile.read(chunk_size)\n            if len(content) < chunk_size:\n                raise exceptions.HttpException(\"Unexpected EOF\")\n            yield content\n            bytes_left -= chunk_size\n    else:\n        bytes_left = limit\n        while bytes_left:\n            chunk_size = min(bytes_left, max_chunk_size)\n            content = rfile.read(chunk_size)\n            if not content:\n                return\n            yield content\n            bytes_left -= chunk_size\n        not_done = rfile.read(1)\n        if not_done:\n            raise exceptions.HttpException(\"HTTP body too large. Limit is {}.\".format(limit))\n\n\ndef connection_close(http_version, headers):\n    \"\"\"\n        Checks the message to see if the client connection should be closed\n        according to RFC 2616 Section 8.1.\n        If we don't have a Connection header, HTTP 1.1 connections are assumed\n        to be persistent.\n    \"\"\"\n    if \"connection\" in headers:\n        tokens = get_header_tokens(headers, \"connection\")\n        if \"close\" in tokens:\n            return True\n        elif \"keep-alive\" in tokens:\n            return False\n\n    return http_version != \"HTTP/1.1\" and http_version != b\"HTTP/1.1\"\n\n\ndef expected_http_body_size(request, response=None):\n    \"\"\"\n        Returns:\n            The expected body length:\n            - a positive integer, if the size is known in advance\n            - None, if the size in unknown in advance (chunked encoding)\n            - -1, if all data should be read until end of stream.\n\n        Raises:\n            exceptions.HttpSyntaxException, if the content length header is invalid\n    \"\"\"\n    # Determine response size according to\n    # http://tools.ietf.org/html/rfc7230#section-3.3\n    if not response:\n        headers = request.headers\n        response_code = None\n        is_request = True\n    else:\n        headers = response.headers\n        response_code = response.status_code\n        is_request = False\n\n    if is_request:\n        if headers.get(\"expect\", \"\").lower() == \"100-continue\":\n            return 0\n    else:\n        if request.method.upper() == \"HEAD\":\n            return 0\n        if 100 <= response_code <= 199:\n            return 0\n        if response_code == 200 and request.method.upper() == \"CONNECT\":\n            return 0\n        if response_code in (204, 304):\n            return 0\n\n    if \"chunked\" in headers.get(\"transfer-encoding\", \"\").lower():\n        return None\n    if \"content-length\" in headers:\n        try:\n            sizes = headers.get_all(\"content-length\")\n            different_content_length_headers = any(x != sizes[0] for x in sizes)\n            if different_content_length_headers:\n                raise exceptions.HttpSyntaxException(\"Conflicting Content Length Headers\")\n            size = int(sizes[0])\n            if size < 0:\n                raise ValueError()\n            return size\n        except ValueError:\n            raise exceptions.HttpSyntaxException(\"Unparseable Content Length\")\n    if is_request:\n        return 0\n    return -1\n\n\ndef _get_first_line(rfile):\n    try:\n        line = rfile.readline()\n        if line == b\"\\r\\n\" or line == b\"\\n\":\n            # Possible leftover from previous message\n            line = rfile.readline()\n    except (exceptions.TcpDisconnect, exceptions.TlsException):\n        raise exceptions.HttpReadDisconnect(\"Remote disconnected\")\n    if not line:\n        raise exceptions.HttpReadDisconnect(\"Remote disconnected\")\n    return line.strip()\n\n\ndef _read_request_line(rfile):\n    try:\n        line = _get_first_line(rfile)\n    except exceptions.HttpReadDisconnect:\n        # We want to provide a better error message.\n        raise exceptions.HttpReadDisconnect(\"Client disconnected\")\n\n    try:\n        method, path, http_version = line.split()\n\n        if path == b\"*\" or path.startswith(b\"/\"):\n            form = \"relative\"\n            scheme, host, port = None, None, None\n        elif method == b\"CONNECT\":\n            form = \"authority\"\n            host, port = _parse_authority_form(path)\n            scheme, path = None, None\n        else:\n            form = \"absolute\"\n            scheme, host, port, path = url.parse(path)\n\n        _check_http_version(http_version)\n    except ValueError:\n        raise exceptions.HttpSyntaxException(\"Bad HTTP request line: {}\".format(line))\n\n    return form, method, scheme, host, port, path, http_version\n\n\ndef _parse_authority_form(hostport):\n    \"\"\"\n        Returns (host, port) if hostport is a valid authority-form host specification.\n        http://tools.ietf.org/html/draft-luotonen-web-proxy-tunneling-01 section 3.1\n\n        Raises:\n            ValueError, if the input is malformed\n    \"\"\"\n    try:\n        host, port = hostport.rsplit(b\":\", 1)\n        if host.startswith(b\"[\") and host.endswith(b\"]\"):\n            host = host[1:-1]\n        port = int(port)\n        if not check.is_valid_host(host) or not check.is_valid_port(port):\n            raise ValueError()\n    except ValueError:\n        raise exceptions.HttpSyntaxException(\"Invalid host specification: {}\".format(hostport))\n\n    return host, port\n\n\ndef _read_response_line(rfile):\n    try:\n        line = _get_first_line(rfile)\n    except exceptions.HttpReadDisconnect:\n        # We want to provide a better error message.\n        raise exceptions.HttpReadDisconnect(\"Server disconnected\")\n\n    try:\n        parts = line.split(None, 2)\n        if len(parts) == 2:  # handle missing message gracefully\n            parts.append(b\"\")\n\n        http_version, status_code, message = parts\n        status_code = int(status_code)\n        _check_http_version(http_version)\n\n    except ValueError:\n        raise exceptions.HttpSyntaxException(\"Bad HTTP response line: {}\".format(line))\n\n    return http_version, status_code, message\n\n\ndef _check_http_version(http_version):\n    if not re.match(br\"^HTTP/\\d\\.\\d$\", http_version):\n        raise exceptions.HttpSyntaxException(\"Unknown HTTP version: {}\".format(http_version))\n\n\ndef _read_headers(rfile):\n    \"\"\"\n        Read a set of headers.\n        Stop once a blank line is reached.\n\n        Returns:\n            A headers object\n\n        Raises:\n            exceptions.HttpSyntaxException\n    \"\"\"\n    ret = []\n    while True:\n        line = rfile.readline()\n        if not line or line == b\"\\r\\n\" or line == b\"\\n\":\n            break\n        if line[0] in b\" \\t\":\n            if not ret:\n                raise exceptions.HttpSyntaxException(\"Invalid headers\")\n            # continued header\n            ret[-1] = (ret[-1][0], ret[-1][1] + b'\\r\\n ' + line.strip())\n        else:\n            try:\n                name, value = line.split(b\":\", 1)\n                value = value.strip()\n                if not name:\n                    raise ValueError()\n                ret.append((name, value))\n            except ValueError:\n                raise exceptions.HttpSyntaxException(\n                    \"Invalid header line: %s\" % repr(line)\n                )\n    return headers.Headers(ret)\n\n\ndef _read_chunked(rfile, limit=sys.maxsize):\n    \"\"\"\n    Read a HTTP body with chunked transfer encoding.\n\n    Args:\n        rfile: the input file\n        limit: A positive integer\n    \"\"\"\n    total = 0\n    while True:\n        line = rfile.readline(128)\n        if line == b\"\":\n            raise exceptions.HttpException(\"Connection closed prematurely\")\n        if line != b\"\\r\\n\" and line != b\"\\n\":\n            try:\n                length = int(line, 16)\n            except ValueError:\n                raise exceptions.HttpSyntaxException(\"Invalid chunked encoding length: {}\".format(line))\n            total += length\n            if total > limit:\n                raise exceptions.HttpException(\n                    \"HTTP Body too large. Limit is {}, \"\n                    \"chunked content longer than {}\".format(limit, total)\n                )\n            chunk = rfile.read(length)\n            suffix = rfile.readline(5)\n            if suffix != b\"\\r\\n\":\n                raise exceptions.HttpSyntaxException(\"Malformed chunked body\")\n            if length == 0:\n                return\n            yield chunk\n", "path": "mitmproxy/net/http/http1/read.py"}]}