refactor to use netlib.http protocols

libmproxy/flow.py

@@ -158,7 +158,7 @@ class StreamLargeBodies(object):
     def run(self, flow, is_request):
         r = flow.request if is_request else flow.response
         code = flow.response.code if flow.response else None
-        expected_size = netlib.http.http1.expected_http_body_size(
+        expected_size = netlib.http.http1.HTTP1Protocol.expected_http_body_size(
             r.headers, is_request, flow.request.method, code
         )
         if not (0 <= expected_size <= self.max_size):

libmproxy/protocol/http.py

@@ -305,6 +305,10 @@ class HTTPRequest(HTTPMessage):
         is_replay=bool
     )
 
+    @property
+    def body(self):
+        return self.content
+
     @classmethod
     def from_state(cls, state):
         f = cls(
@@ -356,11 +360,10 @@ class HTTPRequest(HTTPMessage):
         if hasattr(rfile, "reset_timestamps"):
             rfile.reset_timestamps()
 
-        req = http.http1.read_request(
+        protocol = http.http1.HTTP1Protocol(rfile=rfile, wfile=wfile)
-            rfile,
+        req = protocol.read_request(
             include_body = include_body,
             body_size_limit = body_size_limit,
-            wfile = wfile
         )
 
         if hasattr(rfile, "first_byte_timestamp"):
@@ -377,7 +380,7 @@ class HTTPRequest(HTTPMessage):
             req.path,
             req.httpversion,
             req.headers,
-            req.content,
+            req.body,
             timestamp_start,
             timestamp_end
         )
@@ -726,6 +729,12 @@ class HTTPResponse(HTTPMessage):
         msg=str
     )
 
+
+    @property
+    def body(self):
+        return self.content
+
+
     @classmethod
     def from_state(cls, state):
         f = cls(None, None, None, None, None)
@@ -762,11 +771,12 @@ class HTTPResponse(HTTPMessage):
         if hasattr(rfile, "reset_timestamps"):
             rfile.reset_timestamps()
 
-        resp = http.http1.read_response(
+        protocol = http.http1.HTTP1Protocol(rfile=rfile)
-            rfile,
+        resp = protocol.read_response(
             request_method,
             body_size_limit,
-            include_body=include_body)
+            include_body=include_body
+        )
 
         if hasattr(rfile, "first_byte_timestamp"):
             # more accurate timestamp_start
@@ -782,7 +792,7 @@ class HTTPResponse(HTTPMessage):
             resp.status_code,
             resp.msg,
             resp.headers,
-            resp.content,
+            resp.body,
             timestamp_start,
             timestamp_end
         )
@@ -1046,7 +1056,8 @@ class HTTPHandler(ProtocolHandler):
                 self.c.server_conn.send(request_raw)
                 # Only get the headers at first...
                 flow.response = HTTPResponse.from_stream(
-                    self.c.server_conn.rfile, flow.request.method,
+                    self.c.server_conn.rfile,
+                    flow.request.method,
                     body_size_limit=self.c.config.body_size_limit,
                     include_body=False
                 )
@@ -1083,10 +1094,13 @@ class HTTPHandler(ProtocolHandler):
             if flow.response.stream:
                 flow.response.content = CONTENT_MISSING
             else:
-                flow.response.content = http.http1.read_http_body(
+                protocol = http.http1.HTTP1Protocol(rfile=self.c.server_conn.rfile)
-                    self.c.server_conn.rfile, flow.response.headers,
+                flow.response.content = protocol.read_http_body(
+                    flow.response.headers,
                     self.c.config.body_size_limit,
-                    flow.request.method, flow.response.code, False
+                    flow.request.method,
+                    flow.response.code,
+                    False
                 )
         flow.response.timestamp_end = utils.timestamp()
 
@@ -1287,6 +1301,7 @@ class HTTPHandler(ProtocolHandler):
             if not request.host and flow.server_conn:
                 request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port
 
+
         # Now we can process the request.
         if request.form_in == "authority":
             if self.c.client_conn.ssl_established:
@@ -1420,8 +1435,8 @@ class HTTPHandler(ProtocolHandler):
             h = flow.response._assemble_head(preserve_transfer_encoding=True)
             self.c.client_conn.send(h)
 
-            chunks = http.http1.read_http_body_chunked(
+            protocol = http.http1.HTTP1Protocol(rfile=self.c.server_conn.rfile)
-                self.c.server_conn.rfile,
+            chunks = protocol.read_http_body_chunked(
                 flow.response.headers,
                 self.c.config.body_size_limit,
                 flow.request.method,
@@ -1443,15 +1458,18 @@ class HTTPHandler(ProtocolHandler):
             semantics. Returns True, if so.
         """
         close_connection = (
-            http.http1.connection_close(
+            http.http1.HTTP1Protocol.connection_close(
                 flow.request.httpversion,
-                flow.request.headers) or http.http1.connection_close(
+                flow.request.headers
+            ) or http.http1.HTTP1Protocol.connection_close(
                 flow.response.httpversion,
-                flow.response.headers) or http.http1.expected_http_body_size(
+                flow.response.headers
+            ) or http.http1.HTTP1Protocol.expected_http_body_size(
                 flow.response.headers,
                 False,
                 flow.request.method,
-                flow.response.code) == -1)
+                flow.response.code) == -1
+            )
         if close_connection:
             if flow.request.form_in == "authority" and flow.response.code == 200:
                 # Workaround for

test/test_protocol_http.py

@@ -327,11 +327,11 @@ class TestInvalidRequests(tservers.HTTPProxTest):
         p = self.pathoc()
         r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port))
         assert r.status_code == 400
-        assert "Must not CONNECT on already encrypted connection" in r.content
+        assert "Must not CONNECT on already encrypted connection" in r.body
 
     def test_relative_request(self):
         p = self.pathoc_raw()
         p.connect()
         r = p.request("get:/p/200")
         assert r.status_code == 400
-        assert "Invalid HTTP request form" in r.content
+        assert "Invalid HTTP request form" in r.body

test/test_proxy.py

@@ -31,7 +31,9 @@ class TestServerConnection:
         f.server_conn = sc
         f.request.path = "/p/200:da"
         sc.send(f.request.assemble())
-        assert http.http1.read_response(sc.rfile, f.request.method, 1000)
+
+        protocol = http.http1.HTTP1Protocol(rfile=sc.rfile)
+        assert protocol.read_response(f.request.method, 1000)
         assert self.d.last_log()
 
         sc.finish()

test/test_server.py

@@ -184,6 +184,9 @@ class TcpMixin:
 class AppMixin:
     def test_app(self):
         ret = self.app("/")
+        print(ret)
+        print(ret.status_code)
+        print(ret.content)
         assert ret.status_code == 200
         assert "mitmproxy" in ret.content
 
@@ -767,16 +770,15 @@ class TestStreamRequest(tservers.HTTPProxTest):
             (self.server.urlbase, spec))
         connection.send("\r\n")
 
-        resp = http.http1.read_response(fconn, "GET", None, include_body=False)
+        protocol = http.http1.HTTP1Protocol(rfile=fconn)
+        resp = protocol.read_response("GET", None, include_body=False)
 
         assert resp.headers["Transfer-Encoding"][0] == 'chunked'
         assert resp.status_code == 200
 
         chunks = list(
-            content for _,
+            content for _, content, _ in protocol.read_http_body_chunked(
-                        content,
+                resp.headers, None, "GET", 200, False))
-                        _ in http.http1.read_http_body_chunked(
-                fconn, resp.headers, None, "GET", 200, False))
         assert chunks == ["this", "isatest", ""]
 
         connection.close()