adjust to netlib response changes + docs

docs/dev/models.rst

@@ -3,11 +3,6 @@
 Models
 ======
 
-.. warning::
-    The documentation for models has not been converted to rst yet and **many attributes/features
-    are missing**.
-    Please read the source code instead.
-
 .. automodule:: netlib.http
 
     .. autoclass:: Request
@@ -40,15 +35,23 @@ Models
 
     .. autoclass:: Response
 
-        .. warning:: Docs missing.
+        .. rubric:: Data
-
+        .. autoattribute:: http_version
+        .. autoattribute:: status_code
+        .. autoattribute:: reason
+        .. autoattribute:: headers
+        .. autoattribute:: content
+        .. autoattribute:: timestamp_start
+        .. autoattribute:: timestamp_end
+        .. rubric:: Computed Properties and Convenience Methods
+        .. autoattribute:: text
+        .. autoattribute:: cookies
 
     .. autoclass:: Headers
         :members:
         :special-members:
         :no-undoc-members:
 
-
     .. autoclass:: decoded
 
 .. automodule:: libmproxy.models

libmproxy/console/flowview.py

@@ -168,7 +168,7 @@ class FlowView(tabs.Tabs):
             self.show()
 
     def content_view(self, viewmode, message):
-        if message.body == CONTENT_MISSING:
+        if message.content == CONTENT_MISSING:
             msg, body = "", [urwid.Text([("error", "[content missing]")])]
             return msg, body
         else:
@@ -193,13 +193,13 @@ class FlowView(tabs.Tabs):
 
         try:
             description, lines = contentviews.get_content_view(
-                viewmode, message.body, headers=message.headers
+                viewmode, message.content, headers=message.headers
             )
         except ContentViewException:
             s = "Content viewer failed: \n" + traceback.format_exc()
             signals.add_event(s, "error")
             description, lines = contentviews.get_content_view(
-                contentviews.get("Raw"), message.body, headers=message.headers
+                contentviews.get("Raw"), message.content, headers=message.headers
             )
             description = description.replace("Raw", "Couldn't parse: falling back to Raw")
 

libmproxy/dump.py

@@ -174,15 +174,15 @@ class DumpMaster(flow.FlowMaster):
             )
             self.echo(headers, indent=4)
         if self.o.flow_detail >= 3:
-            if message.body == CONTENT_MISSING:
+            if message.content == CONTENT_MISSING:
                 self.echo("(content missing)", indent=4)
-            elif message.body:
+            elif message.content:
                 self.echo("")
 
                 try:
                     type, lines = contentviews.get_content_view(
                         contentviews.get("Auto"),
-                        message.body,
+                        message.content,
                         headers=message.headers
                     )
                 except ContentViewException:
@@ -190,7 +190,7 @@ class DumpMaster(flow.FlowMaster):
                     self.add_event(s, "debug")
                     type, lines = contentviews.get_content_view(
                         contentviews.get("Raw"),
-                        message.body,
+                        message.content,
                         headers=message.headers
                     )
 
@@ -270,7 +270,7 @@ class DumpMaster(flow.FlowMaster):
         elif 400 <= code < 600:
             code_color = "red"
         code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418))
-        msg = click.style(flow.response.msg, fg=code_color, bold=True)
+        reason = click.style(flow.response.reason, fg=code_color, bold=True)
 
         if flow.response.content == CONTENT_MISSING:
             size = "(content missing)"
@@ -280,11 +280,11 @@ class DumpMaster(flow.FlowMaster):
 
         arrows = click.style("<<", bold=True)
 
-        line = "{replay} {arrows} {code} {msg} {size}".format(
+        line = "{replay} {arrows} {code} {reason} {size}".format(
             replay=replay,
             arrows=arrows,
             code=code,
-            msg=msg,
+            reason=reason,
             size=size
         )
         self.echo(line)

libmproxy/models/http.py

@@ -24,9 +24,9 @@ class MessageMixin(stateobject.StateObject):
     def get_state(self, short=False):
         ret = super(MessageMixin, self).get_state(short)
         if short:
-            if self.body:
+            if self.content:
-                ret["contentLength"] = len(self.body)
+                ret["contentLength"] = len(self.content)
-            elif self.body == CONTENT_MISSING:
+            elif self.content == CONTENT_MISSING:
                 ret["contentLength"] = None
             else:
                 ret["contentLength"] = 0
@@ -39,9 +39,9 @@ class MessageMixin(stateobject.StateObject):
             Doesn't change the message iteself or its headers.
         """
         ce = self.headers.get("content-encoding")
-        if not self.body or ce not in encoding.ENCODINGS:
+        if not self.content or ce not in encoding.ENCODINGS:
-            return self.body
+            return self.content
-        return encoding.decode(ce, self.body)
+        return encoding.decode(ce, self.content)
 
     def decode(self):
         """
@@ -52,12 +52,12 @@ class MessageMixin(stateobject.StateObject):
             Returns True if decoding succeeded, False otherwise.
         """
         ce = self.headers.get("content-encoding")
-        if not self.body or ce not in encoding.ENCODINGS:
+        if not self.content or ce not in encoding.ENCODINGS:
             return False
-        data = encoding.decode(ce, self.body)
+        data = encoding.decode(ce, self.content)
         if data is None:
             return False
-        self.body = data
+        self.content = data
         self.headers.pop("content-encoding", None)
         return True
 
@@ -67,7 +67,7 @@ class MessageMixin(stateobject.StateObject):
             or "identity".
         """
         # FIXME: Error if there's an existing encoding header?
-        self.body = encoding.encode(e, self.body)
+        self.content = encoding.encode(e, self.content)
         self.headers["content-encoding"] = e
 
     def copy(self):
@@ -87,8 +87,8 @@ class MessageMixin(stateobject.StateObject):
             Returns the number of replacements made.
         """
         with decoded(self):
-            self.body, count = utils.safe_subn(
+            self.content, count = utils.safe_subn(
-                pattern, repl, self.body, *args, **kwargs
+                pattern, repl, self.content, *args, **kwargs
             )
         fields = []
         for name, value in self.headers.fields:
@@ -290,9 +290,9 @@ class HTTPResponse(MessageMixin, Response):
             self,
             http_version,
             status_code,
-            msg,
+            reason,
             headers,
-            body,
+            content,
             timestamp_start=None,
             timestamp_end=None,
     ):
@@ -300,9 +300,9 @@ class HTTPResponse(MessageMixin, Response):
             self,
             http_version,
             status_code,
-            msg,
+            reason,
             headers,
-            body,
+            content,
             timestamp_start=timestamp_start,
             timestamp_end=timestamp_end,
         )
@@ -339,9 +339,9 @@ class HTTPResponse(MessageMixin, Response):
         resp = HTTPResponse(
             http_version=response.http_version,
             status_code=response.status_code,
-            msg=response.msg,
+            reason=response.reason,
             headers=response.headers,
-            body=response.body,
+            content=response.content,
             timestamp_start=response.timestamp_start,
             timestamp_end=response.timestamp_end,
         )

libmproxy/protocol/http.py

@@ -51,7 +51,7 @@ class _StreamingHttpLayer(_HttpLayer):
 
     def read_response(self, request):
         response = self.read_response_headers()
-        response.body = b"".join(
+        response.content = b"".join(
             self.read_response_body(request, response)
         )
         return response
@@ -63,10 +63,10 @@ class _StreamingHttpLayer(_HttpLayer):
         raise NotImplementedError()
 
     def send_response(self, response):
-        if response.body == CONTENT_MISSING:
+        if response.content == CONTENT_MISSING:
             raise HttpException("Cannot assemble flow with CONTENT_MISSING")
         self.send_response_headers(response)
-        self.send_response_body(response, [response.body])
+        self.send_response_body(response, [response.content])
 
 
 class Http1Layer(_StreamingHttpLayer):

libmproxy/protocol/tls.py

@@ -7,7 +7,6 @@ from construct import ConstructError
 import six
 from netlib.exceptions import InvalidCertificateException, TcpException, TlsException
 
-from netlib.http import ALPN_PROTO_HTTP1
 from ..contrib.tls._constructs import ClientHello
 from ..exceptions import ProtocolException, TlsProtocolException, ClientHandshakeException
 from .base import Layer
@@ -367,8 +366,8 @@ class TlsLayer(Layer):
         """
 
         # This gets triggered if we haven't established an upstream connection yet.
-        default_alpn = ALPN_PROTO_HTTP1
+        default_alpn = b'http/1.1'
-        # alpn_preference = ALPN_PROTO_H2
+        # alpn_preference = b'h2'
 
         if self.alpn_for_client_connection in options:
             choice = bytes(self.alpn_for_client_connection)

libmproxy/proxy/root_context.py

@@ -6,7 +6,6 @@ import six
 
 from libmproxy.exceptions import ProtocolException
 from netlib.exceptions import TcpException
-from netlib.http import ALPN_PROTO_H2, ALPN_PROTO_HTTP1
 from ..protocol import (
     RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin,
     UpstreamConnectLayer
@@ -85,9 +84,9 @@ class RootContext(object):
         # 5. Check for TLS ALPN (HTTP1/HTTP2)
         if isinstance(top_layer, TlsLayer):
             alpn = top_layer.client_conn.get_alpn_proto_negotiated()
-            if alpn == ALPN_PROTO_H2:
+            if alpn == b'h2':
                 return Http2Layer(top_layer, 'transparent')
-            if alpn == ALPN_PROTO_HTTP1:
+            if alpn == b'http/1.1':
                 return Http1Layer(top_layer, 'transparent')
 
         # 6. Check for raw tcp mode

libmproxy/web/app.py

@@ -29,7 +29,7 @@ class RequestHandler(tornado.web.RequestHandler):
     def json(self):
         if not self.request.headers.get("Content-Type").startswith("application/json"):
             return None
-        return json.loads(self.request.body)
+        return json.loads(self.request.content)
 
     @property
     def state(self):

test/test_dump.py

@@ -38,13 +38,13 @@ def test_strfuncs():
     flow.request.stickycookie = True
     flow.client_conn = mock.MagicMock()
     flow.client_conn.address.host = "foo"
-    flow.response = netlib.tutils.tresp(body=CONTENT_MISSING)
+    flow.response = netlib.tutils.tresp(content=CONTENT_MISSING)
     flow.response.is_replay = True
     flow.response.status_code = 300
     m.echo_flow(flow)
 
 
-    flow = tutils.tflow(resp=netlib.tutils.tresp(body="{"))
+    flow = tutils.tflow(resp=netlib.tutils.tresp(content="{"))
     flow.response.headers["content-type"] = "application/json"
     flow.response.status_code = 400
     m.echo_flow(flow)
@@ -69,7 +69,7 @@ class TestDumpMaster:
         m.handle_clientconnect(f.client_conn)
         m.handle_serverconnect(f.server_conn)
         m.handle_request(f)
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=content))
+        f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=content))
         f = m.handle_response(f)
         m.handle_clientdisconnect(f.client_conn)
         return f

test/test_flow.py

@@ -8,8 +8,8 @@ import mock
 
 import netlib.utils
 from netlib import odict
-from netlib.http import CONTENT_MISSING, HDR_FORM_URLENCODED, Headers
+from netlib.http import CONTENT_MISSING, Headers
-from libmproxy import filt, protocol, controller, tnetstring, flow
+from libmproxy import filt, controller, tnetstring, flow
 from libmproxy.models import Error, Flow, HTTPRequest, HTTPResponse, HTTPFlow, decoded
 from libmproxy.proxy.config import HostMatcher
 from libmproxy.proxy import ProxyConfig
@@ -849,7 +849,7 @@ class TestFlowMaster:
         s = flow.State()
 
         f = tutils.tflow()
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=f.request))
+        f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request))
         pb = [f]
 
         fm = flow.FlowMaster(None, s)
@@ -903,7 +903,7 @@ class TestFlowMaster:
     def test_server_playback_kill(self):
         s = flow.State()
         f = tutils.tflow()
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=f.request))
+        f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request))
         pb = [f]
         fm = flow.FlowMaster(None, s)
         fm.refresh_server_playback = True
@@ -1044,7 +1044,7 @@ class TestRequest:
     def test_getset_form_urlencoded(self):
         d = odict.ODict([("one", "two"), ("three", "four")])
         r = HTTPRequest.wrap(netlib.tutils.treq(content=netlib.utils.urlencode(d.lst)))
-        r.headers["content-type"] = HDR_FORM_URLENCODED
+        r.headers["content-type"] = "application/x-www-form-urlencoded"
         assert r.get_form_urlencoded() == d
 
         d = odict.ODict([("x", "y")])

test/test_protocol_http.py

@@ -46,11 +46,11 @@ class TestInvalidRequests(tservers.HTTPProxTest):
         p = self.pathoc()
         r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port))
         assert r.status_code == 400
-        assert "Invalid HTTP request form" in r.body
+        assert "Invalid HTTP request form" in r.content
 
     def test_relative_request(self):
         p = self.pathoc_raw()
         p.connect()
         r = p.request("get:/p/200")
         assert r.status_code == 400
-        assert "Invalid HTTP request form" in r.body
+        assert "Invalid HTTP request form" in r.content

test/test_server.py

@@ -616,7 +616,7 @@ class MasterRedirectRequest(tservers.TestMaster):
         super(MasterRedirectRequest, self).handle_request(f)
 
     def handle_response(self, f):
-        f.response.body = str(f.client_conn.address.port)
+        f.response.content = str(f.client_conn.address.port)
         f.response.headers["server-conn-id"] = str(f.server_conn.source_address.port)
         super(MasterRedirectRequest, self).handle_response(f)