From 5fd702934c019279264c4efc04086b1879cc5f35 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 21 Jul 2020 17:30:35 +0300 Subject: [PATCH 1/3] Add a msgpack content viewer. --- CHANGELOG | 2 + mitmproxy/contentviews/__init__.py | 3 +- mitmproxy/contentviews/msgpack.py | 36 ++++++++++++++++ setup.py | 1 + test/mitmproxy/contentviews/test_msgpack.py | 46 +++++++++++++++++++++ 5 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 mitmproxy/contentviews/msgpack.py create mode 100644 test/mitmproxy/contentviews/test_msgpack.py diff --git a/CHANGELOG b/CHANGELOG index 64be6495d..74a9a0420 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,8 @@ Unreleased: mitmproxy next ** Full Changelog ** + * Add MsgPack content viewer (@tasn) + * --- TODO: add new PRs above this line --- * ... and various other fixes, documentation improvements, dependency version bumps, etc. diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index f5c31eef6..49c41f2ff 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -22,7 +22,7 @@ from mitmproxy.net import http from mitmproxy.utils import strutils from . import ( auto, raw, hex, json, xml_html, wbxml, javascript, css, - urlencoded, multipart, image, query, protobuf + urlencoded, multipart, image, query, protobuf, msgpack ) from .base import View, KEY_MAX, format_text, format_dict, TViewResult @@ -175,6 +175,7 @@ add(multipart.ViewMultipart()) add(image.ViewImage()) add(query.ViewQuery()) add(protobuf.ViewProtobuf()) +add(msgpack.ViewMsgPack()) __all__ = [ "View", "KEY_MAX", "format_text", "format_dict", "TViewResult", diff --git a/mitmproxy/contentviews/msgpack.py b/mitmproxy/contentviews/msgpack.py new file mode 100644 index 000000000..fc8320fe0 --- /dev/null +++ b/mitmproxy/contentviews/msgpack.py @@ -0,0 +1,36 @@ +import pprint +import re +import typing + +import msgpack + + +from mitmproxy.contentviews import base + +PARSE_ERROR = object() + + +def parse_msgpack(s: bytes) -> typing.Any: + try: + return msgpack.unpackb(s, raw=False) + except (ValueError, msgpack.ExtraData, msgpack.FormatError, msgpack.StackError): + return PARSE_ERROR + + +def format_msgpack(data): + current_line: base.TViewLine = [] + current_line.append(("text", pprint.pformat(data, indent=4))) + yield current_line + + +class ViewMsgPack(base.View): + name = "MsgPack" + content_types = [ + "application/msgpack", + "application/x-msgpack", + ] + + def __call__(self, data, **metadata): + data = parse_msgpack(data) + if data is not PARSE_ERROR: + return "MsgPack", format_msgpack(data) diff --git a/setup.py b/setup.py index ac53d4875..0289a8b70 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,7 @@ setup( "hyperframe>=5.1.0,<6", "kaitaistruct>=0.7,<0.9", "ldap3>=2.6.1,<2.8", + "msgpack>=1.0.0, <1.1.0", "passlib>=1.6.5, <1.8", "protobuf>=3.6.0, <3.12", "pyasn1>=0.3.1,<0.5", diff --git a/test/mitmproxy/contentviews/test_msgpack.py b/test/mitmproxy/contentviews/test_msgpack.py new file mode 100644 index 000000000..d9ce81e80 --- /dev/null +++ b/test/mitmproxy/contentviews/test_msgpack.py @@ -0,0 +1,46 @@ +from hypothesis import given +from hypothesis.strategies import binary + +from msgpack import packb + +from mitmproxy.contentviews import msgpack +from . import full_eval + + +def msgpack_encode(content): + return packb(content, use_bin_type=True) + + +def test_parse_msgpack(): + assert msgpack.parse_msgpack(msgpack_encode({"foo": 1})) + assert msgpack.parse_msgpack(b"aoesuteoahu") is msgpack.PARSE_ERROR + assert msgpack.parse_msgpack(msgpack_encode({"foo": "\xe4\xb8\x96\xe7\x95\x8c"})) + + +def test_format_msgpack(): + assert list(msgpack.format_msgpack({ + "data": [ + "str", + 42, + True, + False, + None, + {}, + [] + ] + })) + + +def test_view_msgpack(): + v = full_eval(msgpack.ViewMsgPack()) + assert v(msgpack_encode({})) + assert not v(b"aoesuteoahu") + assert v(msgpack_encode([1, 2, 3, 4, 5])) + assert v(msgpack_encode({"foo": 3})) + assert v(msgpack_encode({"foo": True, "nullvalue": None})) + + +@given(binary()) +def test_view_msgpack_doesnt_crash(data): + v = full_eval(msgpack.ViewMsgPack()) + v(data) From 008ba5d726f76b101251c87d779e6677673180d8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 21 Jul 2020 18:27:19 +0300 Subject: [PATCH 2/3] Improve the formatting of the msgpack content viewer. --- mitmproxy/contentviews/msgpack.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/mitmproxy/contentviews/msgpack.py b/mitmproxy/contentviews/msgpack.py index fc8320fe0..821cccc23 100644 --- a/mitmproxy/contentviews/msgpack.py +++ b/mitmproxy/contentviews/msgpack.py @@ -17,10 +17,25 @@ def parse_msgpack(s: bytes) -> typing.Any: return PARSE_ERROR +def pretty(value, htchar=" ", lfchar="\n", indent=0): + nlch = lfchar + htchar * (indent + 1) + if type(value) is dict: + items = [ + nlch + repr(key) + ": " + pretty(value[key], htchar, lfchar, indent + 1) + for key in value + ] + return "{%s}" % (",".join(items) + lfchar + htchar * indent) + elif type(value) is list: + items = [ + nlch + pretty(item, htchar, lfchar, indent + 1) + for item in value + ] + return "[%s]" % (",".join(items) + lfchar + htchar * indent) + else: + return repr(value) + def format_msgpack(data): - current_line: base.TViewLine = [] - current_line.append(("text", pprint.pformat(data, indent=4))) - yield current_line + return base.format_text(pretty(data)) class ViewMsgPack(base.View): From 1bca5597f6787293a71b3433e413ade934fed7c2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Wed, 22 Jul 2020 16:24:05 +0300 Subject: [PATCH 3/3] Fix lint errors --- mitmproxy/contentviews/msgpack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mitmproxy/contentviews/msgpack.py b/mitmproxy/contentviews/msgpack.py index 821cccc23..fec1f60c2 100644 --- a/mitmproxy/contentviews/msgpack.py +++ b/mitmproxy/contentviews/msgpack.py @@ -1,5 +1,3 @@ -import pprint -import re import typing import msgpack @@ -34,6 +32,7 @@ def pretty(value, htchar=" ", lfchar="\n", indent=0): else: return repr(value) + def format_msgpack(data): return base.format_text(pretty(data))