From 1b6ea5caf3ab7c30cc4a822b81ad8288efbae70d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 26 Sep 2015 17:41:14 +0200 Subject: [PATCH] adjust to netlib response changes + docs --- docs/dev/models.rst | 19 +++++++++-------- libmproxy/console/flowview.py | 6 +++--- libmproxy/dump.py | 14 ++++++------- libmproxy/models/http.py | 36 ++++++++++++++++----------------- libmproxy/protocol/http.py | 6 +++--- libmproxy/protocol/tls.py | 5 ++--- libmproxy/proxy/root_context.py | 5 ++--- libmproxy/web/app.py | 2 +- test/test_dump.py | 6 +++--- test/test_flow.py | 10 ++++----- test/test_protocol_http.py | 4 ++-- test/test_server.py | 2 +- 12 files changed, 58 insertions(+), 57 deletions(-) diff --git a/docs/dev/models.rst b/docs/dev/models.rst index 617bcb1f4..b09c8baef 100644 --- a/docs/dev/models.rst +++ b/docs/dev/models.rst @@ -3,11 +3,6 @@ Models ====== -.. warning:: - The documentation for models has not been converted to rst yet and **many attributes/features - are missing**. - Please read the source code instead. - .. automodule:: netlib.http .. autoclass:: Request @@ -40,15 +35,23 @@ Models .. autoclass:: Response - .. warning:: Docs missing. - + .. rubric:: Data + .. autoattribute:: http_version + .. autoattribute:: status_code + .. autoattribute:: reason + .. autoattribute:: headers + .. autoattribute:: content + .. autoattribute:: timestamp_start + .. autoattribute:: timestamp_end + .. rubric:: Computed Properties and Convenience Methods + .. autoattribute:: text + .. autoattribute:: cookies .. autoclass:: Headers :members: :special-members: :no-undoc-members: - .. autoclass:: decoded .. automodule:: libmproxy.models diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 5979bff31..5271db4f6 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -168,7 +168,7 @@ class FlowView(tabs.Tabs): self.show() def content_view(self, viewmode, message): - if message.body == CONTENT_MISSING: + if message.content == CONTENT_MISSING: msg, body = "", [urwid.Text([("error", "[content missing]")])] return msg, body else: @@ -193,13 +193,13 @@ class FlowView(tabs.Tabs): try: description, lines = contentviews.get_content_view( - viewmode, message.body, headers=message.headers + viewmode, message.content, headers=message.headers ) except ContentViewException: s = "Content viewer failed: \n" + traceback.format_exc() signals.add_event(s, "error") description, lines = contentviews.get_content_view( - contentviews.get("Raw"), message.body, headers=message.headers + contentviews.get("Raw"), message.content, headers=message.headers ) description = description.replace("Raw", "Couldn't parse: falling back to Raw") diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 43af953cf..d2b130f1d 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -174,15 +174,15 @@ class DumpMaster(flow.FlowMaster): ) self.echo(headers, indent=4) if self.o.flow_detail >= 3: - if message.body == CONTENT_MISSING: + if message.content == CONTENT_MISSING: self.echo("(content missing)", indent=4) - elif message.body: + elif message.content: self.echo("") try: type, lines = contentviews.get_content_view( contentviews.get("Auto"), - message.body, + message.content, headers=message.headers ) except ContentViewException: @@ -190,7 +190,7 @@ class DumpMaster(flow.FlowMaster): self.add_event(s, "debug") type, lines = contentviews.get_content_view( contentviews.get("Raw"), - message.body, + message.content, headers=message.headers ) @@ -270,7 +270,7 @@ class DumpMaster(flow.FlowMaster): elif 400 <= code < 600: code_color = "red" code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418)) - msg = click.style(flow.response.msg, fg=code_color, bold=True) + reason = click.style(flow.response.reason, fg=code_color, bold=True) if flow.response.content == CONTENT_MISSING: size = "(content missing)" @@ -280,11 +280,11 @@ class DumpMaster(flow.FlowMaster): arrows = click.style("<<", bold=True) - line = "{replay} {arrows} {code} {msg} {size}".format( + line = "{replay} {arrows} {code} {reason} {size}".format( replay=replay, arrows=arrows, code=code, - msg=msg, + reason=reason, size=size ) self.echo(line) diff --git a/libmproxy/models/http.py b/libmproxy/models/http.py index 101df7e63..d820f4dc4 100644 --- a/libmproxy/models/http.py +++ b/libmproxy/models/http.py @@ -24,9 +24,9 @@ class MessageMixin(stateobject.StateObject): def get_state(self, short=False): ret = super(MessageMixin, self).get_state(short) if short: - if self.body: - ret["contentLength"] = len(self.body) - elif self.body == CONTENT_MISSING: + if self.content: + ret["contentLength"] = len(self.content) + elif self.content == CONTENT_MISSING: ret["contentLength"] = None else: ret["contentLength"] = 0 @@ -39,9 +39,9 @@ class MessageMixin(stateobject.StateObject): Doesn't change the message iteself or its headers. """ ce = self.headers.get("content-encoding") - if not self.body or ce not in encoding.ENCODINGS: - return self.body - return encoding.decode(ce, self.body) + if not self.content or ce not in encoding.ENCODINGS: + return self.content + return encoding.decode(ce, self.content) def decode(self): """ @@ -52,12 +52,12 @@ class MessageMixin(stateobject.StateObject): Returns True if decoding succeeded, False otherwise. """ ce = self.headers.get("content-encoding") - if not self.body or ce not in encoding.ENCODINGS: + if not self.content or ce not in encoding.ENCODINGS: return False - data = encoding.decode(ce, self.body) + data = encoding.decode(ce, self.content) if data is None: return False - self.body = data + self.content = data self.headers.pop("content-encoding", None) return True @@ -67,7 +67,7 @@ class MessageMixin(stateobject.StateObject): or "identity". """ # FIXME: Error if there's an existing encoding header? - self.body = encoding.encode(e, self.body) + self.content = encoding.encode(e, self.content) self.headers["content-encoding"] = e def copy(self): @@ -87,8 +87,8 @@ class MessageMixin(stateobject.StateObject): Returns the number of replacements made. """ with decoded(self): - self.body, count = utils.safe_subn( - pattern, repl, self.body, *args, **kwargs + self.content, count = utils.safe_subn( + pattern, repl, self.content, *args, **kwargs ) fields = [] for name, value in self.headers.fields: @@ -290,9 +290,9 @@ class HTTPResponse(MessageMixin, Response): self, http_version, status_code, - msg, + reason, headers, - body, + content, timestamp_start=None, timestamp_end=None, ): @@ -300,9 +300,9 @@ class HTTPResponse(MessageMixin, Response): self, http_version, status_code, - msg, + reason, headers, - body, + content, timestamp_start=timestamp_start, timestamp_end=timestamp_end, ) @@ -339,9 +339,9 @@ class HTTPResponse(MessageMixin, Response): resp = HTTPResponse( http_version=response.http_version, status_code=response.status_code, - msg=response.msg, + reason=response.reason, headers=response.headers, - body=response.body, + content=response.content, timestamp_start=response.timestamp_start, timestamp_end=response.timestamp_end, ) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 21c2a72c3..d4a7622cc 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -51,7 +51,7 @@ class _StreamingHttpLayer(_HttpLayer): def read_response(self, request): response = self.read_response_headers() - response.body = b"".join( + response.content = b"".join( self.read_response_body(request, response) ) return response @@ -63,10 +63,10 @@ class _StreamingHttpLayer(_HttpLayer): raise NotImplementedError() def send_response(self, response): - if response.body == CONTENT_MISSING: + if response.content == CONTENT_MISSING: raise HttpException("Cannot assemble flow with CONTENT_MISSING") self.send_response_headers(response) - self.send_response_body(response, [response.body]) + self.send_response_body(response, [response.content]) class Http1Layer(_StreamingHttpLayer): diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index d144e0815..9764fb226 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -7,7 +7,6 @@ from construct import ConstructError import six from netlib.exceptions import InvalidCertificateException, TcpException, TlsException -from netlib.http import ALPN_PROTO_HTTP1 from ..contrib.tls._constructs import ClientHello from ..exceptions import ProtocolException, TlsProtocolException, ClientHandshakeException from .base import Layer @@ -367,8 +366,8 @@ class TlsLayer(Layer): """ # This gets triggered if we haven't established an upstream connection yet. - default_alpn = ALPN_PROTO_HTTP1 - # alpn_preference = ALPN_PROTO_H2 + default_alpn = b'http/1.1' + # alpn_preference = b'h2' if self.alpn_for_client_connection in options: choice = bytes(self.alpn_for_client_connection) diff --git a/libmproxy/proxy/root_context.py b/libmproxy/proxy/root_context.py index 307d0c4b8..f62b0c8ec 100644 --- a/libmproxy/proxy/root_context.py +++ b/libmproxy/proxy/root_context.py @@ -6,7 +6,6 @@ import six from libmproxy.exceptions import ProtocolException from netlib.exceptions import TcpException -from netlib.http import ALPN_PROTO_H2, ALPN_PROTO_HTTP1 from ..protocol import ( RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin, UpstreamConnectLayer @@ -85,9 +84,9 @@ class RootContext(object): # 5. Check for TLS ALPN (HTTP1/HTTP2) if isinstance(top_layer, TlsLayer): alpn = top_layer.client_conn.get_alpn_proto_negotiated() - if alpn == ALPN_PROTO_H2: + if alpn == b'h2': return Http2Layer(top_layer, 'transparent') - if alpn == ALPN_PROTO_HTTP1: + if alpn == b'http/1.1': return Http1Layer(top_layer, 'transparent') # 6. Check for raw tcp mode diff --git a/libmproxy/web/app.py b/libmproxy/web/app.py index 5c80584dc..58dc77e7c 100644 --- a/libmproxy/web/app.py +++ b/libmproxy/web/app.py @@ -29,7 +29,7 @@ class RequestHandler(tornado.web.RequestHandler): def json(self): if not self.request.headers.get("Content-Type").startswith("application/json"): return None - return json.loads(self.request.body) + return json.loads(self.request.content) @property def state(self): diff --git a/test/test_dump.py b/test/test_dump.py index 9586a3042..a903a651f 100644 --- a/test/test_dump.py +++ b/test/test_dump.py @@ -38,13 +38,13 @@ def test_strfuncs(): flow.request.stickycookie = True flow.client_conn = mock.MagicMock() flow.client_conn.address.host = "foo" - flow.response = netlib.tutils.tresp(body=CONTENT_MISSING) + flow.response = netlib.tutils.tresp(content=CONTENT_MISSING) flow.response.is_replay = True flow.response.status_code = 300 m.echo_flow(flow) - flow = tutils.tflow(resp=netlib.tutils.tresp(body="{")) + flow = tutils.tflow(resp=netlib.tutils.tresp(content="{")) flow.response.headers["content-type"] = "application/json" flow.response.status_code = 400 m.echo_flow(flow) @@ -69,7 +69,7 @@ class TestDumpMaster: m.handle_clientconnect(f.client_conn) m.handle_serverconnect(f.server_conn) m.handle_request(f) - f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=content)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=content)) f = m.handle_response(f) m.handle_clientdisconnect(f.client_conn) return f diff --git a/test/test_flow.py b/test/test_flow.py index 0cd455768..ca862bdaf 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -8,8 +8,8 @@ import mock import netlib.utils from netlib import odict -from netlib.http import CONTENT_MISSING, HDR_FORM_URLENCODED, Headers -from libmproxy import filt, protocol, controller, tnetstring, flow +from netlib.http import CONTENT_MISSING, Headers +from libmproxy import filt, controller, tnetstring, flow from libmproxy.models import Error, Flow, HTTPRequest, HTTPResponse, HTTPFlow, decoded from libmproxy.proxy.config import HostMatcher from libmproxy.proxy import ProxyConfig @@ -849,7 +849,7 @@ class TestFlowMaster: s = flow.State() f = tutils.tflow() - f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=f.request)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] fm = flow.FlowMaster(None, s) @@ -903,7 +903,7 @@ class TestFlowMaster: def test_server_playback_kill(self): s = flow.State() f = tutils.tflow() - f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=f.request)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] fm = flow.FlowMaster(None, s) fm.refresh_server_playback = True @@ -1044,7 +1044,7 @@ class TestRequest: def test_getset_form_urlencoded(self): d = odict.ODict([("one", "two"), ("three", "four")]) r = HTTPRequest.wrap(netlib.tutils.treq(content=netlib.utils.urlencode(d.lst))) - r.headers["content-type"] = HDR_FORM_URLENCODED + r.headers["content-type"] = "application/x-www-form-urlencoded" assert r.get_form_urlencoded() == d d = odict.ODict([("x", "y")]) diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index 5ddb5b5b8..4158adda4 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -46,11 +46,11 @@ class TestInvalidRequests(tservers.HTTPProxTest): p = self.pathoc() r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port)) assert r.status_code == 400 - assert "Invalid HTTP request form" in r.body + assert "Invalid HTTP request form" in r.content def test_relative_request(self): p = self.pathoc_raw() p.connect() r = p.request("get:/p/200") assert r.status_code == 400 - assert "Invalid HTTP request form" in r.body + assert "Invalid HTTP request form" in r.content diff --git a/test/test_server.py b/test/test_server.py index 2a09d2e53..9488595fa 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -616,7 +616,7 @@ class MasterRedirectRequest(tservers.TestMaster): super(MasterRedirectRequest, self).handle_request(f) def handle_response(self, f): - f.response.body = str(f.client_conn.address.port) + f.response.content = str(f.client_conn.address.port) f.response.headers["server-conn-id"] = str(f.server_conn.source_address.port) super(MasterRedirectRequest, self).handle_response(f)