We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Crash of har_dump.py
I sometimes get either `b'' is not JSON serializable` or  `Object of type 'bytes' is not JSON serializable`

I used mitmproxy to extract such a flow and uploaded it [here](https://gist.github.com/Junkern/495d4c7c1b572d7749f0291adfb0dd15)

##### Steps to reproduce the problem:

1.
2.
3.


##### Any other comments? What have you tried so far?

It seems, that the error mostly occurs for `POST` and `HEAD` requests. Sometimes with an empty response body, sometimes not. Seldomly a `GET` requests triggers this error

##### System information
mitmproxy 1.0.2
macOs Sierra 10.12.3

Edit: Updated the Gist link to point to a working binary gist

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `examples/complex/har_dump.py`
Content:
```
1 """
2 This inline script can be used to dump flows as HAR files.
3 """
4 
5 
6 import json
7 import sys
8 import base64
9 import zlib
10 
11 from datetime import datetime
12 import pytz
13 
14 import mitmproxy
15 
16 from mitmproxy import version
17 from mitmproxy.utils import strutils
18 from mitmproxy.net.http import cookies
19 
20 HAR = {}
21 
22 # A list of server seen till now is maintained so we can avoid
23 # using 'connect' time for entries that use an existing connection.
24 SERVERS_SEEN = set()
25 
26 
27 def start():
28     """
29         Called once on script startup before any other events.
30     """
31     if len(sys.argv) != 2:
32         raise ValueError(
33             'Usage: -s "har_dump.py filename" '
34             '(- will output to stdout, filenames ending with .zhar '
35             'will result in compressed har)'
36         )
37 
38     HAR.update({
39         "log": {
40             "version": "1.2",
41             "creator": {
42                 "name": "mitmproxy har_dump",
43                 "version": "0.1",
44                 "comment": "mitmproxy version %s" % version.MITMPROXY
45             },
46             "entries": []
47         }
48     })
49 
50 
51 def response(flow):
52     """
53        Called when a server response has been received.
54     """
55 
56     # -1 indicates that these values do not apply to current request
57     ssl_time = -1
58     connect_time = -1
59 
60     if flow.server_conn and flow.server_conn not in SERVERS_SEEN:
61         connect_time = (flow.server_conn.timestamp_tcp_setup -
62                         flow.server_conn.timestamp_start)
63 
64         if flow.server_conn.timestamp_ssl_setup is not None:
65             ssl_time = (flow.server_conn.timestamp_ssl_setup -
66                         flow.server_conn.timestamp_tcp_setup)
67 
68         SERVERS_SEEN.add(flow.server_conn)
69 
70     # Calculate raw timings from timestamps. DNS timings can not be calculated
71     # for lack of a way to measure it. The same goes for HAR blocked.
72     # mitmproxy will open a server connection as soon as it receives the host
73     # and port from the client connection. So, the time spent waiting is actually
74     # spent waiting between request.timestamp_end and response.timestamp_start
75     # thus it correlates to HAR wait instead.
76     timings_raw = {
77         'send': flow.request.timestamp_end - flow.request.timestamp_start,
78         'receive': flow.response.timestamp_end - flow.response.timestamp_start,
79         'wait': flow.response.timestamp_start - flow.request.timestamp_end,
80         'connect': connect_time,
81         'ssl': ssl_time,
82     }
83 
84     # HAR timings are integers in ms, so we re-encode the raw timings to that format.
85     timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])
86 
87     # full_time is the sum of all timings.
88     # Timings set to -1 will be ignored as per spec.
89     full_time = sum(v for v in timings.values() if v > -1)
90 
91     started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))
92 
93     # Response body size and encoding
94     response_body_size = len(flow.response.raw_content)
95     response_body_decoded_size = len(flow.response.content)
96     response_body_compression = response_body_decoded_size - response_body_size
97 
98     entry = {
99         "startedDateTime": started_date_time,
100         "time": full_time,
101         "request": {
102             "method": flow.request.method,
103             "url": flow.request.url,
104             "httpVersion": flow.request.http_version,
105             "cookies": format_request_cookies(flow.request.cookies.fields),
106             "headers": name_value(flow.request.headers),
107             "queryString": name_value(flow.request.query or {}),
108             "headersSize": len(str(flow.request.headers)),
109             "bodySize": len(flow.request.content),
110         },
111         "response": {
112             "status": flow.response.status_code,
113             "statusText": flow.response.reason,
114             "httpVersion": flow.response.http_version,
115             "cookies": format_response_cookies(flow.response.cookies.fields),
116             "headers": name_value(flow.response.headers),
117             "content": {
118                 "size": response_body_size,
119                 "compression": response_body_compression,
120                 "mimeType": flow.response.headers.get('Content-Type', '')
121             },
122             "redirectURL": flow.response.headers.get('Location', ''),
123             "headersSize": len(str(flow.response.headers)),
124             "bodySize": response_body_size,
125         },
126         "cache": {},
127         "timings": timings,
128     }
129 
130     # Store binary data as base64
131     if strutils.is_mostly_bin(flow.response.content):
132         entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode()
133         entry["response"]["content"]["encoding"] = "base64"
134     else:
135         entry["response"]["content"]["text"] = flow.response.get_text(strict=False)
136 
137     if flow.request.method in ["POST", "PUT", "PATCH"]:
138         params = [
139             {"name": a, "value": b}
140             for a, b in flow.request.urlencoded_form.items(multi=True)
141         ]
142         entry["request"]["postData"] = {
143             "mimeType": flow.request.headers.get("Content-Type", ""),
144             "text": flow.request.get_text(strict=False),
145             "params": params
146         }
147 
148     if flow.server_conn.connected():
149         entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])
150 
151     HAR["log"]["entries"].append(entry)
152 
153 
154 def done():
155     """
156         Called once on script shutdown, after any other events.
157     """
158     dump_file = sys.argv[1]
159 
160     json_dump = json.dumps(HAR, indent=2)  # type: str
161 
162     if dump_file == '-':
163         mitmproxy.ctx.log(json_dump)
164     else:
165         raw = json_dump.encode()  # type: bytes
166         if dump_file.endswith('.zhar'):
167             raw = zlib.compress(raw, 9)
168 
169         with open(dump_file, "wb") as f:
170             f.write(raw)
171 
172         mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
173 
174 
175 def format_datetime(dt):
176     return dt.replace(tzinfo=pytz.timezone("UTC")).isoformat()
177 
178 
179 def format_cookies(cookie_list):
180     rv = []
181 
182     for name, value, attrs in cookie_list:
183         cookie_har = {
184             "name": name,
185             "value": value,
186         }
187 
188         # HAR only needs some attributes
189         for key in ["path", "domain", "comment"]:
190             if key in attrs:
191                 cookie_har[key] = attrs[key]
192 
193         # These keys need to be boolean!
194         for key in ["httpOnly", "secure"]:
195             cookie_har[key] = bool(key in attrs)
196 
197         # Expiration time needs to be formatted
198         expire_ts = cookies.get_expiration_ts(attrs)
199         if expire_ts is not None:
200             cookie_har["expires"] = format_datetime(datetime.fromtimestamp(expire_ts))
201 
202         rv.append(cookie_har)
203 
204     return rv
205 
206 
207 def format_request_cookies(fields):
208     return format_cookies(cookies.group_cookies(fields))
209 
210 
211 def format_response_cookies(fields):
212     return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields)
213 
214 
215 def name_value(obj):
216     """
217         Convert (key, value) pairs to HAR format.
218     """
219     return [{"name": k, "value": v} for k, v in obj.items()]
220
```

Path: `mitmproxy/net/http/encoding.py`
Content:
```
1 """
2 Utility functions for decoding response bodies.
3 """
4 
5 import codecs
6 import collections
7 from io import BytesIO
8 
9 import gzip
10 import zlib
11 import brotli
12 
13 from typing import Union
14 
15 
16 # We have a shared single-element cache for encoding and decoding.
17 # This is quite useful in practice, e.g.
18 # flow.request.content = flow.request.content.replace(b"foo", b"bar")
19 # does not require an .encode() call if content does not contain b"foo"
20 CachedDecode = collections.namedtuple("CachedDecode", "encoded encoding errors decoded")
21 _cache = CachedDecode(None, None, None, None)
22 
23 
24 def decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]:
25     """
26     Decode the given input object
27 
28     Returns:
29         The decoded value
30 
31     Raises:
32         ValueError, if decoding fails.
33     """
34     if len(encoded) == 0:
35         return encoded
36 
37     global _cache
38     cached = (
39         isinstance(encoded, bytes) and
40         _cache.encoded == encoded and
41         _cache.encoding == encoding and
42         _cache.errors == errors
43     )
44     if cached:
45         return _cache.decoded
46     try:
47         try:
48             decoded = custom_decode[encoding](encoded)
49         except KeyError:
50             decoded = codecs.decode(encoded, encoding, errors)
51         if encoding in ("gzip", "deflate", "br"):
52             _cache = CachedDecode(encoded, encoding, errors, decoded)
53         return decoded
54     except TypeError:
55         raise
56     except Exception as e:
57         raise ValueError("{} when decoding {} with {}: {}".format(
58             type(e).__name__,
59             repr(encoded)[:10],
60             repr(encoding),
61             repr(e),
62         ))
63 
64 
65 def encode(decoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]:
66     """
67     Encode the given input object
68 
69     Returns:
70         The encoded value
71 
72     Raises:
73         ValueError, if encoding fails.
74     """
75     if len(decoded) == 0:
76         return decoded
77 
78     global _cache
79     cached = (
80         isinstance(decoded, bytes) and
81         _cache.decoded == decoded and
82         _cache.encoding == encoding and
83         _cache.errors == errors
84     )
85     if cached:
86         return _cache.encoded
87     try:
88         try:
89             value = decoded
90             if isinstance(value, str):
91                 value = decoded.encode()
92             encoded = custom_encode[encoding](value)
93         except KeyError:
94             encoded = codecs.encode(decoded, encoding, errors)
95         if encoding in ("gzip", "deflate", "br"):
96             _cache = CachedDecode(encoded, encoding, errors, decoded)
97         return encoded
98     except TypeError:
99         raise
100     except Exception as e:
101         raise ValueError("{} when encoding {} with {}: {}".format(
102             type(e).__name__,
103             repr(decoded)[:10],
104             repr(encoding),
105             repr(e),
106         ))
107 
108 
109 def identity(content):
110     """
111         Returns content unchanged. Identity is the default value of
112         Accept-Encoding headers.
113     """
114     return content
115 
116 
117 def decode_gzip(content):
118     gfile = gzip.GzipFile(fileobj=BytesIO(content))
119     return gfile.read()
120 
121 
122 def encode_gzip(content):
123     s = BytesIO()
124     gf = gzip.GzipFile(fileobj=s, mode='wb')
125     gf.write(content)
126     gf.close()
127     return s.getvalue()
128 
129 
130 def decode_brotli(content):
131     return brotli.decompress(content)
132 
133 
134 def encode_brotli(content):
135     return brotli.compress(content)
136 
137 
138 def decode_deflate(content):
139     """
140         Returns decompressed data for DEFLATE. Some servers may respond with
141         compressed data without a zlib header or checksum. An undocumented
142         feature of zlib permits the lenient decompression of data missing both
143         values.
144 
145         http://bugs.python.org/issue5784
146     """
147     try:
148         return zlib.decompress(content)
149     except zlib.error:
150         return zlib.decompress(content, -15)
151 
152 
153 def encode_deflate(content):
154     """
155         Returns compressed content, always including zlib header and checksum.
156     """
157     return zlib.compress(content)
158 
159 
160 custom_decode = {
161     "none": identity,
162     "identity": identity,
163     "gzip": decode_gzip,
164     "deflate": decode_deflate,
165     "br": decode_brotli,
166 }
167 custom_encode = {
168     "none": identity,
169     "identity": identity,
170     "gzip": encode_gzip,
171     "deflate": encode_deflate,
172     "br": encode_brotli,
173 }
174 
175 __all__ = ["encode", "decode"]
176
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py
--- a/examples/complex/har_dump.py
+++ b/examples/complex/har_dump.py
@@ -7,6 +7,7 @@
 import sys
 import base64
 import zlib
+import os
 
 from datetime import datetime
 import pytz
@@ -166,7 +167,7 @@
         if dump_file.endswith('.zhar'):
             raw = zlib.compress(raw, 9)
 
-        with open(dump_file, "wb") as f:
+        with open(os.path.expanduser(dump_file), "wb") as f:
             f.write(raw)
 
         mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
diff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py
--- a/mitmproxy/net/http/encoding.py
+++ b/mitmproxy/net/http/encoding.py
@@ -31,8 +31,8 @@
     Raises:
         ValueError, if decoding fails.
     """
-    if len(encoded) == 0:
-        return encoded
+    if encoded is None:
+        return None
 
     global _cache
     cached = (
@@ -72,8 +72,8 @@
     Raises:
         ValueError, if encoding fails.
     """
-    if len(decoded) == 0:
-        return decoded
+    if decoded is None:
+        return None
 
     global _cache
     cached = (
@@ -86,10 +86,7 @@
         return _cache.encoded
     try:
         try:
-            value = decoded
-            if isinstance(value, str):
-                value = decoded.encode()
-            encoded = custom_encode[encoding](value)
+            encoded = custom_encode[encoding](decoded)
         except KeyError:
             encoded = codecs.encode(decoded, encoding, errors)
         if encoding in ("gzip", "deflate", "br"):
@@ -114,12 +111,14 @@
     return content
 
 
-def decode_gzip(content):
+def decode_gzip(content: bytes) -> bytes:
+    if not content:
+        return b""
     gfile = gzip.GzipFile(fileobj=BytesIO(content))
     return gfile.read()
 
 
-def encode_gzip(content):
+def encode_gzip(content: bytes) -> bytes:
     s = BytesIO()
     gf = gzip.GzipFile(fileobj=s, mode='wb')
     gf.write(content)
@@ -127,15 +126,17 @@
     return s.getvalue()
 
 
-def decode_brotli(content):
+def decode_brotli(content: bytes) -> bytes:
+    if not content:
+        return b""
     return brotli.decompress(content)
 
 
-def encode_brotli(content):
+def encode_brotli(content: bytes) -> bytes:
     return brotli.compress(content)
 
 
-def decode_deflate(content):
+def decode_deflate(content: bytes) -> bytes:
     """
         Returns decompressed data for DEFLATE. Some servers may respond with
         compressed data without a zlib header or checksum. An undocumented
@@ -144,13 +145,15 @@
 
         http://bugs.python.org/issue5784
     """
+    if not content:
+        return b""
     try:
         return zlib.decompress(content)
     except zlib.error:
         return zlib.decompress(content, -15)
 
 
-def encode_deflate(content):
+def encode_deflate(content: bytes) -> bytes:
     """
         Returns compressed content, always including zlib header and checksum.
     """

{"golden_diff": "diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py\n--- a/examples/complex/har_dump.py\n+++ b/examples/complex/har_dump.py\n@@ -7,6 +7,7 @@\n import sys\n import base64\n import zlib\n+import os\n \n from datetime import datetime\n import pytz\n@@ -166,7 +167,7 @@\n         if dump_file.endswith('.zhar'):\n             raw = zlib.compress(raw, 9)\n \n-        with open(dump_file, \"wb\") as f:\n+        with open(os.path.expanduser(dump_file), \"wb\") as f:\n             f.write(raw)\n \n         mitmproxy.ctx.log(\"HAR dump finished (wrote %s bytes to file)\" % len(json_dump))\ndiff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py\n--- a/mitmproxy/net/http/encoding.py\n+++ b/mitmproxy/net/http/encoding.py\n@@ -31,8 +31,8 @@\n     Raises:\n         ValueError, if decoding fails.\n     \"\"\"\n-    if len(encoded) == 0:\n-        return encoded\n+    if encoded is None:\n+        return None\n \n     global _cache\n     cached = (\n@@ -72,8 +72,8 @@\n     Raises:\n         ValueError, if encoding fails.\n     \"\"\"\n-    if len(decoded) == 0:\n-        return decoded\n+    if decoded is None:\n+        return None\n \n     global _cache\n     cached = (\n@@ -86,10 +86,7 @@\n         return _cache.encoded\n     try:\n         try:\n-            value = decoded\n-            if isinstance(value, str):\n-                value = decoded.encode()\n-            encoded = custom_encode[encoding](value)\n+            encoded = custom_encode[encoding](decoded)\n         except KeyError:\n             encoded = codecs.encode(decoded, encoding, errors)\n         if encoding in (\"gzip\", \"deflate\", \"br\"):\n@@ -114,12 +111,14 @@\n     return content\n \n \n-def decode_gzip(content):\n+def decode_gzip(content: bytes) -> bytes:\n+    if not content:\n+        return b\"\"\n     gfile = gzip.GzipFile(fileobj=BytesIO(content))\n     return gfile.read()\n \n \n-def encode_gzip(content):\n+def encode_gzip(content: bytes) -> bytes:\n     s = BytesIO()\n     gf = gzip.GzipFile(fileobj=s, mode='wb')\n     gf.write(content)\n@@ -127,15 +126,17 @@\n     return s.getvalue()\n \n \n-def decode_brotli(content):\n+def decode_brotli(content: bytes) -> bytes:\n+    if not content:\n+        return b\"\"\n     return brotli.decompress(content)\n \n \n-def encode_brotli(content):\n+def encode_brotli(content: bytes) -> bytes:\n     return brotli.compress(content)\n \n \n-def decode_deflate(content):\n+def decode_deflate(content: bytes) -> bytes:\n     \"\"\"\n         Returns decompressed data for DEFLATE. Some servers may respond with\n         compressed data without a zlib header or checksum. An undocumented\n@@ -144,13 +145,15 @@\n \n         http://bugs.python.org/issue5784\n     \"\"\"\n+    if not content:\n+        return b\"\"\n     try:\n         return zlib.decompress(content)\n     except zlib.error:\n         return zlib.decompress(content, -15)\n \n \n-def encode_deflate(content):\n+def encode_deflate(content: bytes) -> bytes:\n     \"\"\"\n         Returns compressed content, always including zlib header and checksum.\n     \"\"\"\n", "issue": "Crash of har_dump.py\nI sometimes get either `b'' is not JSON serializable` or  `Object of type 'bytes' is not JSON serializable`\r\n\r\nI used mitmproxy to extract such a flow and uploaded it [here](https://gist.github.com/Junkern/495d4c7c1b572d7749f0291adfb0dd15)\r\n\r\n##### Steps to reproduce the problem:\r\n\r\n1.\r\n2.\r\n3.\r\n\r\n\r\n##### Any other comments? What have you tried so far?\r\n\r\nIt seems, that the error mostly occurs for `POST` and `HEAD` requests. Sometimes with an empty response body, sometimes not. Seldomly a `GET` requests triggers this error\r\n\r\n##### System information\r\nmitmproxy 1.0.2\r\nmacOs Sierra 10.12.3\r\n\r\nEdit: Updated the Gist link to point to a working binary gist\n", "before_files": [{"content": "\"\"\"\nThis inline script can be used to dump flows as HAR files.\n\"\"\"\n\n\nimport json\nimport sys\nimport base64\nimport zlib\n\nfrom datetime import datetime\nimport pytz\n\nimport mitmproxy\n\nfrom mitmproxy import version\nfrom mitmproxy.utils import strutils\nfrom mitmproxy.net.http import cookies\n\nHAR = {}\n\n# A list of server seen till now is maintained so we can avoid\n# using 'connect' time for entries that use an existing connection.\nSERVERS_SEEN = set()\n\n\ndef start():\n    \"\"\"\n        Called once on script startup before any other events.\n    \"\"\"\n    if len(sys.argv) != 2:\n        raise ValueError(\n            'Usage: -s \"har_dump.py filename\" '\n            '(- will output to stdout, filenames ending with .zhar '\n            'will result in compressed har)'\n        )\n\n    HAR.update({\n        \"log\": {\n            \"version\": \"1.2\",\n            \"creator\": {\n                \"name\": \"mitmproxy har_dump\",\n                \"version\": \"0.1\",\n                \"comment\": \"mitmproxy version %s\" % version.MITMPROXY\n            },\n            \"entries\": []\n        }\n    })\n\n\ndef response(flow):\n    \"\"\"\n       Called when a server response has been received.\n    \"\"\"\n\n    # -1 indicates that these values do not apply to current request\n    ssl_time = -1\n    connect_time = -1\n\n    if flow.server_conn and flow.server_conn not in SERVERS_SEEN:\n        connect_time = (flow.server_conn.timestamp_tcp_setup -\n                        flow.server_conn.timestamp_start)\n\n        if flow.server_conn.timestamp_ssl_setup is not None:\n            ssl_time = (flow.server_conn.timestamp_ssl_setup -\n                        flow.server_conn.timestamp_tcp_setup)\n\n        SERVERS_SEEN.add(flow.server_conn)\n\n    # Calculate raw timings from timestamps. DNS timings can not be calculated\n    # for lack of a way to measure it. The same goes for HAR blocked.\n    # mitmproxy will open a server connection as soon as it receives the host\n    # and port from the client connection. So, the time spent waiting is actually\n    # spent waiting between request.timestamp_end and response.timestamp_start\n    # thus it correlates to HAR wait instead.\n    timings_raw = {\n        'send': flow.request.timestamp_end - flow.request.timestamp_start,\n        'receive': flow.response.timestamp_end - flow.response.timestamp_start,\n        'wait': flow.response.timestamp_start - flow.request.timestamp_end,\n        'connect': connect_time,\n        'ssl': ssl_time,\n    }\n\n    # HAR timings are integers in ms, so we re-encode the raw timings to that format.\n    timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])\n\n    # full_time is the sum of all timings.\n    # Timings set to -1 will be ignored as per spec.\n    full_time = sum(v for v in timings.values() if v > -1)\n\n    started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))\n\n    # Response body size and encoding\n    response_body_size = len(flow.response.raw_content)\n    response_body_decoded_size = len(flow.response.content)\n    response_body_compression = response_body_decoded_size - response_body_size\n\n    entry = {\n        \"startedDateTime\": started_date_time,\n        \"time\": full_time,\n        \"request\": {\n            \"method\": flow.request.method,\n            \"url\": flow.request.url,\n            \"httpVersion\": flow.request.http_version,\n            \"cookies\": format_request_cookies(flow.request.cookies.fields),\n            \"headers\": name_value(flow.request.headers),\n            \"queryString\": name_value(flow.request.query or {}),\n            \"headersSize\": len(str(flow.request.headers)),\n            \"bodySize\": len(flow.request.content),\n        },\n        \"response\": {\n            \"status\": flow.response.status_code,\n            \"statusText\": flow.response.reason,\n            \"httpVersion\": flow.response.http_version,\n            \"cookies\": format_response_cookies(flow.response.cookies.fields),\n            \"headers\": name_value(flow.response.headers),\n            \"content\": {\n                \"size\": response_body_size,\n                \"compression\": response_body_compression,\n                \"mimeType\": flow.response.headers.get('Content-Type', '')\n            },\n            \"redirectURL\": flow.response.headers.get('Location', ''),\n            \"headersSize\": len(str(flow.response.headers)),\n            \"bodySize\": response_body_size,\n        },\n        \"cache\": {},\n        \"timings\": timings,\n    }\n\n    # Store binary data as base64\n    if strutils.is_mostly_bin(flow.response.content):\n        entry[\"response\"][\"content\"][\"text\"] = base64.b64encode(flow.response.content).decode()\n        entry[\"response\"][\"content\"][\"encoding\"] = \"base64\"\n    else:\n        entry[\"response\"][\"content\"][\"text\"] = flow.response.get_text(strict=False)\n\n    if flow.request.method in [\"POST\", \"PUT\", \"PATCH\"]:\n        params = [\n            {\"name\": a, \"value\": b}\n            for a, b in flow.request.urlencoded_form.items(multi=True)\n        ]\n        entry[\"request\"][\"postData\"] = {\n            \"mimeType\": flow.request.headers.get(\"Content-Type\", \"\"),\n            \"text\": flow.request.get_text(strict=False),\n            \"params\": params\n        }\n\n    if flow.server_conn.connected():\n        entry[\"serverIPAddress\"] = str(flow.server_conn.ip_address.address[0])\n\n    HAR[\"log\"][\"entries\"].append(entry)\n\n\ndef done():\n    \"\"\"\n        Called once on script shutdown, after any other events.\n    \"\"\"\n    dump_file = sys.argv[1]\n\n    json_dump = json.dumps(HAR, indent=2)  # type: str\n\n    if dump_file == '-':\n        mitmproxy.ctx.log(json_dump)\n    else:\n        raw = json_dump.encode()  # type: bytes\n        if dump_file.endswith('.zhar'):\n            raw = zlib.compress(raw, 9)\n\n        with open(dump_file, \"wb\") as f:\n            f.write(raw)\n\n        mitmproxy.ctx.log(\"HAR dump finished (wrote %s bytes to file)\" % len(json_dump))\n\n\ndef format_datetime(dt):\n    return dt.replace(tzinfo=pytz.timezone(\"UTC\")).isoformat()\n\n\ndef format_cookies(cookie_list):\n    rv = []\n\n    for name, value, attrs in cookie_list:\n        cookie_har = {\n            \"name\": name,\n            \"value\": value,\n        }\n\n        # HAR only needs some attributes\n        for key in [\"path\", \"domain\", \"comment\"]:\n            if key in attrs:\n                cookie_har[key] = attrs[key]\n\n        # These keys need to be boolean!\n        for key in [\"httpOnly\", \"secure\"]:\n            cookie_har[key] = bool(key in attrs)\n\n        # Expiration time needs to be formatted\n        expire_ts = cookies.get_expiration_ts(attrs)\n        if expire_ts is not None:\n            cookie_har[\"expires\"] = format_datetime(datetime.fromtimestamp(expire_ts))\n\n        rv.append(cookie_har)\n\n    return rv\n\n\ndef format_request_cookies(fields):\n    return format_cookies(cookies.group_cookies(fields))\n\n\ndef format_response_cookies(fields):\n    return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields)\n\n\ndef name_value(obj):\n    \"\"\"\n        Convert (key, value) pairs to HAR format.\n    \"\"\"\n    return [{\"name\": k, \"value\": v} for k, v in obj.items()]\n", "path": "examples/complex/har_dump.py"}, {"content": "\"\"\"\nUtility functions for decoding response bodies.\n\"\"\"\n\nimport codecs\nimport collections\nfrom io import BytesIO\n\nimport gzip\nimport zlib\nimport brotli\n\nfrom typing import Union\n\n\n# We have a shared single-element cache for encoding and decoding.\n# This is quite useful in practice, e.g.\n# flow.request.content = flow.request.content.replace(b\"foo\", b\"bar\")\n# does not require an .encode() call if content does not contain b\"foo\"\nCachedDecode = collections.namedtuple(\"CachedDecode\", \"encoded encoding errors decoded\")\n_cache = CachedDecode(None, None, None, None)\n\n\ndef decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]:\n    \"\"\"\n    Decode the given input object\n\n    Returns:\n        The decoded value\n\n    Raises:\n        ValueError, if decoding fails.\n    \"\"\"\n    if len(encoded) == 0:\n        return encoded\n\n    global _cache\n    cached = (\n        isinstance(encoded, bytes) and\n        _cache.encoded == encoded and\n        _cache.encoding == encoding and\n        _cache.errors == errors\n    )\n    if cached:\n        return _cache.decoded\n    try:\n        try:\n            decoded = custom_decode[encoding](encoded)\n        except KeyError:\n            decoded = codecs.decode(encoded, encoding, errors)\n        if encoding in (\"gzip\", \"deflate\", \"br\"):\n            _cache = CachedDecode(encoded, encoding, errors, decoded)\n        return decoded\n    except TypeError:\n        raise\n    except Exception as e:\n        raise ValueError(\"{} when decoding {} with {}: {}\".format(\n            type(e).__name__,\n            repr(encoded)[:10],\n            repr(encoding),\n            repr(e),\n        ))\n\n\ndef encode(decoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]:\n    \"\"\"\n    Encode the given input object\n\n    Returns:\n        The encoded value\n\n    Raises:\n        ValueError, if encoding fails.\n    \"\"\"\n    if len(decoded) == 0:\n        return decoded\n\n    global _cache\n    cached = (\n        isinstance(decoded, bytes) and\n        _cache.decoded == decoded and\n        _cache.encoding == encoding and\n        _cache.errors == errors\n    )\n    if cached:\n        return _cache.encoded\n    try:\n        try:\n            value = decoded\n            if isinstance(value, str):\n                value = decoded.encode()\n            encoded = custom_encode[encoding](value)\n        except KeyError:\n            encoded = codecs.encode(decoded, encoding, errors)\n        if encoding in (\"gzip\", \"deflate\", \"br\"):\n            _cache = CachedDecode(encoded, encoding, errors, decoded)\n        return encoded\n    except TypeError:\n        raise\n    except Exception as e:\n        raise ValueError(\"{} when encoding {} with {}: {}\".format(\n            type(e).__name__,\n            repr(decoded)[:10],\n            repr(encoding),\n            repr(e),\n        ))\n\n\ndef identity(content):\n    \"\"\"\n        Returns content unchanged. Identity is the default value of\n        Accept-Encoding headers.\n    \"\"\"\n    return content\n\n\ndef decode_gzip(content):\n    gfile = gzip.GzipFile(fileobj=BytesIO(content))\n    return gfile.read()\n\n\ndef encode_gzip(content):\n    s = BytesIO()\n    gf = gzip.GzipFile(fileobj=s, mode='wb')\n    gf.write(content)\n    gf.close()\n    return s.getvalue()\n\n\ndef decode_brotli(content):\n    return brotli.decompress(content)\n\n\ndef encode_brotli(content):\n    return brotli.compress(content)\n\n\ndef decode_deflate(content):\n    \"\"\"\n        Returns decompressed data for DEFLATE. Some servers may respond with\n        compressed data without a zlib header or checksum. An undocumented\n        feature of zlib permits the lenient decompression of data missing both\n        values.\n\n        http://bugs.python.org/issue5784\n    \"\"\"\n    try:\n        return zlib.decompress(content)\n    except zlib.error:\n        return zlib.decompress(content, -15)\n\n\ndef encode_deflate(content):\n    \"\"\"\n        Returns compressed content, always including zlib header and checksum.\n    \"\"\"\n    return zlib.compress(content)\n\n\ncustom_decode = {\n    \"none\": identity,\n    \"identity\": identity,\n    \"gzip\": decode_gzip,\n    \"deflate\": decode_deflate,\n    \"br\": decode_brotli,\n}\ncustom_encode = {\n    \"none\": identity,\n    \"identity\": identity,\n    \"gzip\": encode_gzip,\n    \"deflate\": encode_deflate,\n    \"br\": encode_brotli,\n}\n\n__all__ = [\"encode\", \"decode\"]\n", "path": "mitmproxy/net/http/encoding.py"}], "after_files": [{"content": "\"\"\"\nThis inline script can be used to dump flows as HAR files.\n\"\"\"\n\n\nimport json\nimport sys\nimport base64\nimport zlib\nimport os\n\nfrom datetime import datetime\nimport pytz\n\nimport mitmproxy\n\nfrom mitmproxy import version\nfrom mitmproxy.utils import strutils\nfrom mitmproxy.net.http import cookies\n\nHAR = {}\n\n# A list of server seen till now is maintained so we can avoid\n# using 'connect' time for entries that use an existing connection.\nSERVERS_SEEN = set()\n\n\ndef start():\n    \"\"\"\n        Called once on script startup before any other events.\n    \"\"\"\n    if len(sys.argv) != 2:\n        raise ValueError(\n            'Usage: -s \"har_dump.py filename\" '\n            '(- will output to stdout, filenames ending with .zhar '\n            'will result in compressed har)'\n        )\n\n    HAR.update({\n        \"log\": {\n            \"version\": \"1.2\",\n            \"creator\": {\n                \"name\": \"mitmproxy har_dump\",\n                \"version\": \"0.1\",\n                \"comment\": \"mitmproxy version %s\" % version.MITMPROXY\n            },\n            \"entries\": []\n        }\n    })\n\n\ndef response(flow):\n    \"\"\"\n       Called when a server response has been received.\n    \"\"\"\n\n    # -1 indicates that these values do not apply to current request\n    ssl_time = -1\n    connect_time = -1\n\n    if flow.server_conn and flow.server_conn not in SERVERS_SEEN:\n        connect_time = (flow.server_conn.timestamp_tcp_setup -\n                        flow.server_conn.timestamp_start)\n\n        if flow.server_conn.timestamp_ssl_setup is not None:\n            ssl_time = (flow.server_conn.timestamp_ssl_setup -\n                        flow.server_conn.timestamp_tcp_setup)\n\n        SERVERS_SEEN.add(flow.server_conn)\n\n    # Calculate raw timings from timestamps. DNS timings can not be calculated\n    # for lack of a way to measure it. The same goes for HAR blocked.\n    # mitmproxy will open a server connection as soon as it receives the host\n    # and port from the client connection. So, the time spent waiting is actually\n    # spent waiting between request.timestamp_end and response.timestamp_start\n    # thus it correlates to HAR wait instead.\n    timings_raw = {\n        'send': flow.request.timestamp_end - flow.request.timestamp_start,\n        'receive': flow.response.timestamp_end - flow.response.timestamp_start,\n        'wait': flow.response.timestamp_start - flow.request.timestamp_end,\n        'connect': connect_time,\n        'ssl': ssl_time,\n    }\n\n    # HAR timings are integers in ms, so we re-encode the raw timings to that format.\n    timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])\n\n    # full_time is the sum of all timings.\n    # Timings set to -1 will be ignored as per spec.\n    full_time = sum(v for v in timings.values() if v > -1)\n\n    started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))\n\n    # Response body size and encoding\n    response_body_size = len(flow.response.raw_content)\n    response_body_decoded_size = len(flow.response.content)\n    response_body_compression = response_body_decoded_size - response_body_size\n\n    entry = {\n        \"startedDateTime\": started_date_time,\n        \"time\": full_time,\n        \"request\": {\n            \"method\": flow.request.method,\n            \"url\": flow.request.url,\n            \"httpVersion\": flow.request.http_version,\n            \"cookies\": format_request_cookies(flow.request.cookies.fields),\n            \"headers\": name_value(flow.request.headers),\n            \"queryString\": name_value(flow.request.query or {}),\n            \"headersSize\": len(str(flow.request.headers)),\n            \"bodySize\": len(flow.request.content),\n        },\n        \"response\": {\n            \"status\": flow.response.status_code,\n            \"statusText\": flow.response.reason,\n            \"httpVersion\": flow.response.http_version,\n            \"cookies\": format_response_cookies(flow.response.cookies.fields),\n            \"headers\": name_value(flow.response.headers),\n            \"content\": {\n                \"size\": response_body_size,\n                \"compression\": response_body_compression,\n                \"mimeType\": flow.response.headers.get('Content-Type', '')\n            },\n            \"redirectURL\": flow.response.headers.get('Location', ''),\n            \"headersSize\": len(str(flow.response.headers)),\n            \"bodySize\": response_body_size,\n        },\n        \"cache\": {},\n        \"timings\": timings,\n    }\n\n    # Store binary data as base64\n    if strutils.is_mostly_bin(flow.response.content):\n        entry[\"response\"][\"content\"][\"text\"] = base64.b64encode(flow.response.content).decode()\n        entry[\"response\"][\"content\"][\"encoding\"] = \"base64\"\n    else:\n        entry[\"response\"][\"content\"][\"text\"] = flow.response.get_text(strict=False)\n\n    if flow.request.method in [\"POST\", \"PUT\", \"PATCH\"]:\n        params = [\n            {\"name\": a, \"value\": b}\n            for a, b in flow.request.urlencoded_form.items(multi=True)\n        ]\n        entry[\"request\"][\"postData\"] = {\n            \"mimeType\": flow.request.headers.get(\"Content-Type\", \"\"),\n            \"text\": flow.request.get_text(strict=False),\n            \"params\": params\n        }\n\n    if flow.server_conn.connected():\n        entry[\"serverIPAddress\"] = str(flow.server_conn.ip_address.address[0])\n\n    HAR[\"log\"][\"entries\"].append(entry)\n\n\ndef done():\n    \"\"\"\n        Called once on script shutdown, after any other events.\n    \"\"\"\n    dump_file = sys.argv[1]\n\n    json_dump = json.dumps(HAR, indent=2)  # type: str\n\n    if dump_file == '-':\n        mitmproxy.ctx.log(json_dump)\n    else:\n        raw = json_dump.encode()  # type: bytes\n        if dump_file.endswith('.zhar'):\n            raw = zlib.compress(raw, 9)\n\n        with open(os.path.expanduser(dump_file), \"wb\") as f:\n            f.write(raw)\n\n        mitmproxy.ctx.log(\"HAR dump finished (wrote %s bytes to file)\" % len(json_dump))\n\n\ndef format_datetime(dt):\n    return dt.replace(tzinfo=pytz.timezone(\"UTC\")).isoformat()\n\n\ndef format_cookies(cookie_list):\n    rv = []\n\n    for name, value, attrs in cookie_list:\n        cookie_har = {\n            \"name\": name,\n            \"value\": value,\n        }\n\n        # HAR only needs some attributes\n        for key in [\"path\", \"domain\", \"comment\"]:\n            if key in attrs:\n                cookie_har[key] = attrs[key]\n\n        # These keys need to be boolean!\n        for key in [\"httpOnly\", \"secure\"]:\n            cookie_har[key] = bool(key in attrs)\n\n        # Expiration time needs to be formatted\n        expire_ts = cookies.get_expiration_ts(attrs)\n        if expire_ts is not None:\n            cookie_har[\"expires\"] = format_datetime(datetime.fromtimestamp(expire_ts))\n\n        rv.append(cookie_har)\n\n    return rv\n\n\ndef format_request_cookies(fields):\n    return format_cookies(cookies.group_cookies(fields))\n\n\ndef format_response_cookies(fields):\n    return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields)\n\n\ndef name_value(obj):\n    \"\"\"\n        Convert (key, value) pairs to HAR format.\n    \"\"\"\n    return [{\"name\": k, \"value\": v} for k, v in obj.items()]\n", "path": "examples/complex/har_dump.py"}, {"content": "\"\"\"\nUtility functions for decoding response bodies.\n\"\"\"\n\nimport codecs\nimport collections\nfrom io import BytesIO\n\nimport gzip\nimport zlib\nimport brotli\n\nfrom typing import Union\n\n\n# We have a shared single-element cache for encoding and decoding.\n# This is quite useful in practice, e.g.\n# flow.request.content = flow.request.content.replace(b\"foo\", b\"bar\")\n# does not require an .encode() call if content does not contain b\"foo\"\nCachedDecode = collections.namedtuple(\"CachedDecode\", \"encoded encoding errors decoded\")\n_cache = CachedDecode(None, None, None, None)\n\n\ndef decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]:\n    \"\"\"\n    Decode the given input object\n\n    Returns:\n        The decoded value\n\n    Raises:\n        ValueError, if decoding fails.\n    \"\"\"\n    if encoded is None:\n        return None\n\n    global _cache\n    cached = (\n        isinstance(encoded, bytes) and\n        _cache.encoded == encoded and\n        _cache.encoding == encoding and\n        _cache.errors == errors\n    )\n    if cached:\n        return _cache.decoded\n    try:\n        try:\n            decoded = custom_decode[encoding](encoded)\n        except KeyError:\n            decoded = codecs.decode(encoded, encoding, errors)\n        if encoding in (\"gzip\", \"deflate\", \"br\"):\n            _cache = CachedDecode(encoded, encoding, errors, decoded)\n        return decoded\n    except TypeError:\n        raise\n    except Exception as e:\n        raise ValueError(\"{} when decoding {} with {}: {}\".format(\n            type(e).__name__,\n            repr(encoded)[:10],\n            repr(encoding),\n            repr(e),\n        ))\n\n\ndef encode(decoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]:\n    \"\"\"\n    Encode the given input object\n\n    Returns:\n        The encoded value\n\n    Raises:\n        ValueError, if encoding fails.\n    \"\"\"\n    if decoded is None:\n        return None\n\n    global _cache\n    cached = (\n        isinstance(decoded, bytes) and\n        _cache.decoded == decoded and\n        _cache.encoding == encoding and\n        _cache.errors == errors\n    )\n    if cached:\n        return _cache.encoded\n    try:\n        try:\n            encoded = custom_encode[encoding](decoded)\n        except KeyError:\n            encoded = codecs.encode(decoded, encoding, errors)\n        if encoding in (\"gzip\", \"deflate\", \"br\"):\n            _cache = CachedDecode(encoded, encoding, errors, decoded)\n        return encoded\n    except TypeError:\n        raise\n    except Exception as e:\n        raise ValueError(\"{} when encoding {} with {}: {}\".format(\n            type(e).__name__,\n            repr(decoded)[:10],\n            repr(encoding),\n            repr(e),\n        ))\n\n\ndef identity(content):\n    \"\"\"\n        Returns content unchanged. Identity is the default value of\n        Accept-Encoding headers.\n    \"\"\"\n    return content\n\n\ndef decode_gzip(content: bytes) -> bytes:\n    if not content:\n        return b\"\"\n    gfile = gzip.GzipFile(fileobj=BytesIO(content))\n    return gfile.read()\n\n\ndef encode_gzip(content: bytes) -> bytes:\n    s = BytesIO()\n    gf = gzip.GzipFile(fileobj=s, mode='wb')\n    gf.write(content)\n    gf.close()\n    return s.getvalue()\n\n\ndef decode_brotli(content: bytes) -> bytes:\n    if not content:\n        return b\"\"\n    return brotli.decompress(content)\n\n\ndef encode_brotli(content: bytes) -> bytes:\n    return brotli.compress(content)\n\n\ndef decode_deflate(content: bytes) -> bytes:\n    \"\"\"\n        Returns decompressed data for DEFLATE. Some servers may respond with\n        compressed data without a zlib header or checksum. An undocumented\n        feature of zlib permits the lenient decompression of data missing both\n        values.\n\n        http://bugs.python.org/issue5784\n    \"\"\"\n    if not content:\n        return b\"\"\n    try:\n        return zlib.decompress(content)\n    except zlib.error:\n        return zlib.decompress(content, -15)\n\n\ndef encode_deflate(content: bytes) -> bytes:\n    \"\"\"\n        Returns compressed content, always including zlib header and checksum.\n    \"\"\"\n    return zlib.compress(content)\n\n\ncustom_decode = {\n    \"none\": identity,\n    \"identity\": identity,\n    \"gzip\": decode_gzip,\n    \"deflate\": decode_deflate,\n    \"br\": decode_brotli,\n}\ncustom_encode = {\n    \"none\": identity,\n    \"identity\": identity,\n    \"gzip\": encode_gzip,\n    \"deflate\": encode_deflate,\n    \"br\": encode_brotli,\n}\n\n__all__ = [\"encode\", \"decode\"]\n", "path": "mitmproxy/net/http/encoding.py"}]}