Fix crash while streaming

Found using fuzzing. Reproduction with pathoc, given "mitmproxy -s" and
pathod running on 9999:

get:'http://localhost:9999/p/':s'200:b\'foo\':h\'Content-Length\'=\'3\'':i58,'\x1a':r

 return flow.FlowMaster.run(self)
   File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/controller.py", line 111, in run
     self.tick(self.masterq, 0.01)
   File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/flow.py", line 613, in tick
     return controller.Master.tick(self, q, timeout)
   File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/controller.py", line 101, in tick
     self.handle(*msg)
   File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/controller.py", line 118, in handle
     m(obj)
   File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/flow.py", line 738, in handle_responseheaders
     self.stream_large_bodies.run(f, False)
   File "/Users/aldo/mitmproxy/mitmproxy/libmproxy/flow.py", line 155, in run
     r.headers, is_request, flow.request.method, code
   File "/Users/aldo/mitmproxy/mitmproxy/netlib/http.py", line 401, in expected_http_body_size
     raise HttpError(400 if is_request else 502, "Invalid content-length header: %s" % headers["content-length"])
 netlib.http.HttpError: Invalid content-length header: ['\x1a3']

libmproxy/cmdline.py

@@ -248,14 +248,18 @@ def common_options(parser):
         "--stream",
         action="store", dest="stream_large_bodies", default=None,
         metavar="SIZE",
-        help="Stream data to the client if response body exceeds the given threshold. "
+        help="""
-             "If streamed, the body will not be stored in any way. Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
+        Stream data to the client if response body exceeds the given threshold.
+        If streamed, the body will not be stored in any way. Understands k/m/g
+        suffixes, i.e. 3m for 3 megabytes.
+         """
     )
 
     group = parser.add_argument_group("Proxy Options")
-    # We could make a mutually exclusive group out of -R, -U, -T, but we don't do that because
+    # We could make a mutually exclusive group out of -R, -U, -T, but we don't
-    #  - --upstream-server should be in that group as well, but it's already in a different group.
+    # do that because  - --upstream-server should be in that group as well, but
-    #  - our own error messages are more helpful
+    # it's already in a different group.  - our own error messages are more
+    # helpful
     group.add_argument(
         "-b",
         action="store", type=str, dest="addr", default='',
@@ -265,11 +269,14 @@ def common_options(parser):
         "-I", "--ignore",
         action="append", type=str, dest="ignore_hosts", default=[],
         metavar="HOST",
-        help="Ignore host and forward all traffic without processing it. "
+        help="""
-             "In transparent mode, it is recommended to use an IP address (range), not the hostname. "
+            Ignore host and forward all traffic without processing it. In
-             "In regular mode, only SSL traffic is ignored and the hostname should be used. "
+            transparent mode, it is recommended to use an IP address (range),
-             "The supplied value is interpreted as a regular expression and matched on the ip or the hostname. "
+            not the hostname. In regular mode, only SSL traffic is ignored and
-             "Can be passed multiple times. "
+            the hostname should be used. The supplied value is interpreted as a
+            regular expression and matched on the ip or the hostname. Can be
+            passed multiple times.
+        """
     )
     group.add_argument(
         "--tcp",

libmproxy/flow.py

@@ -738,8 +738,12 @@ class FlowMaster(controller.Master):
     def handle_responseheaders(self, f):
         self.run_script_hook("responseheaders", f)
 
-        if self.stream_large_bodies:
+        try:
-            self.stream_large_bodies.run(f, False)
+            if self.stream_large_bodies:
+                self.stream_large_bodies.run(f, False)
+        except netlib.http.HttpError:
+            f.reply(protocol.KILL)
+            return
 
         f.reply()
         return f

libmproxy/protocol/http.py

@@ -18,6 +18,10 @@ HDR_FORM_URLENCODED = "application/x-www-form-urlencoded"
 CONTENT_MISSING = 0
 
 
+class KillSignal(Exception):
+    pass
+
+
 def get_line(fp):
     """
     Get a line, possibly preceded by a blank.
@@ -1001,19 +1005,21 @@ class HTTPHandler(ProtocolHandler):
 
         # call the appropriate script hook - this is an opportunity for an
         # inline script to set flow.stream = True
-        self.c.channel.ask("responseheaders", flow)
+        flow = self.c.channel.ask("responseheaders", flow)
-
+        if flow == KILL:
-        # now get the rest of the request body, if body still needs to be read
+            raise KillSignal
-        # but not streaming this response
-        if flow.response.stream:
-            flow.response.content = CONTENT_MISSING
         else:
-            flow.response.content = http.read_http_body(
+            # now get the rest of the request body, if body still needs to be
-                self.c.server_conn.rfile, flow.response.headers,
+            # read but not streaming this response
-                self.c.config.body_size_limit,
+            if flow.response.stream:
-                flow.request.method, flow.response.code, False
+                flow.response.content = CONTENT_MISSING
-            )
+            else:
-            flow.response.timestamp_end = utils.timestamp()
+                flow.response.content = http.read_http_body(
+                    self.c.server_conn.rfile, flow.response.headers,
+                    self.c.config.body_size_limit,
+                    flow.request.method, flow.response.code, False
+                )
+        flow.response.timestamp_end = utils.timestamp()
 
     def handle_flow(self):
         flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live)
@@ -1092,8 +1098,16 @@ class HTTPHandler(ProtocolHandler):
             flow.live.restore_server()
 
             return True  # Next flow please.
-        except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e:
+        except (
+                HttpAuthenticationError,
+                http.HttpError,
+                proxy.ProxyError,
+                tcp.NetLibError,
+        ), e:
             self.handle_error(e, flow)
+        except KillSignal:
+            self.c.log("Connection killed", "info")
+            flow.live = None
         finally:
             flow.live = None  # Connection is not live anymore.
         return False

test/fuzzing/straight_stream

@@ -1,4 +1,6 @@
 
 mitmdump: $MITMDUMP -q --stream 1 
 pathod: $PATHOD -q
-pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS  localhost:8080  ./straight_stream_patterns
+#pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS  localhost:8080  ./straight_stream_patterns
+pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS  localhost:8080  /tmp/err
+