From 469bd32582f764f9a29607efa4f5b04bd87961fb Mon Sep 17 00:00:00 2001 From: Nikita Stupin <18281368+nikitastupin@users.noreply.github.com> Date: Fri, 1 May 2020 20:57:27 +0300 Subject: [PATCH 1/8] Integrate contentviews to TCP flow details --- mitmproxy/contentviews/__init__.py | 10 ++++++++++ mitmproxy/tools/console/flowview.py | 31 ++++++++++++----------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index fc38543dc..063b8cfe1 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -116,6 +116,16 @@ def get_message_content_view(viewname, message, flow): return description, lines, error +def get_tcp_content_view(viewname: str, data: bytes): + viewmode = get(viewname) + if not viewmode: + viewmode = get("auto") + + description, lines, error = get_content_view(viewmode, data) + + return description, lines, error + + def get_content_view(viewmode: View, data: bytes, **metadata): """ Args: diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 3fef70cee..65206d51d 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -115,26 +115,21 @@ class FlowDetails(tabs.Tabs): if not flow.messages: return searchable.Searchable([urwid.Text(("highlight", "No messages."))]) - from_client = None - messages = [] - for message in flow.messages: - if message.from_client is not from_client: - messages.append(message.content) - from_client = message.from_client - else: - messages[-1] += message.content + viewmode = self.master.commands.call("console.flowview.mode") - from_client = flow.messages[0].from_client parts = [] - for message in messages: - parts.append( - ( - "head" if from_client else "key", - message - ) - ) - from_client = not from_client - return searchable.Searchable([urwid.Text(parts)]) + for message in flow.messages: + _, lines, _ = contentviews.get_tcp_content_view(viewmode, message.content) + + for line in lines: + if message.from_client: + line.insert(0, "--> ") + else: + line.insert(0, "<-- ") + + parts.append(urwid.Text(line)) + + return searchable.Searchable(parts) def view_details(self): return flowdetailview.flowdetails(self.view, self.flow) From 7aaa6217839c5351e508caa73adf76e1b1a3b678 Mon Sep 17 00:00:00 2001 From: Nikita Stupin <18281368+nikitastupin@users.noreply.github.com> Date: Sat, 2 May 2020 12:25:59 +0300 Subject: [PATCH 2/8] Merge adjacent TCP "messages" --- mitmproxy/tools/console/flowview.py | 31 ++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 65206d51d..7c6258cfe 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -117,19 +117,40 @@ class FlowDetails(tabs.Tabs): viewmode = self.master.commands.call("console.flowview.mode") - parts = [] + # Merge adjacent TCP "messages". For detailed explanation of this code block see: + # https://github.com/mitmproxy/mitmproxy/pull/3970/files/469bd32582f764f9a29607efa4f5b04bd87961fb#r418670880 + merged_messages = [] for message in flow.messages: - _, lines, _ = contentviews.get_tcp_content_view(viewmode, message.content) + + if not merged_messages: + merged_messages.append({ + "content": message.content, + "from_client": message.from_client, + }) + continue + + if merged_messages[-1]["from_client"] == message.from_client: + merged_messages[-1]["content"] += message.content + else: + merged_messages.append({ + "content": message.content, + "from_client": message.from_client, + }) + + + widget_lines = [] + for message in merged_messages: + _, lines, _ = contentviews.get_tcp_content_view(viewmode, message["content"]) for line in lines: - if message.from_client: + if message["from_client"]: line.insert(0, "--> ") else: line.insert(0, "<-- ") - parts.append(urwid.Text(line)) + widget_lines.append(urwid.Text(line)) - return searchable.Searchable(parts) + return searchable.Searchable(widget_lines) def view_details(self): return flowdetailview.flowdetails(self.view, self.flow) From 2927d66d01f3a8798aeb20b7e4a238a5be98aff7 Mon Sep 17 00:00:00 2001 From: Nikita Stupin <18281368+nikitastupin@users.noreply.github.com> Date: Sat, 2 May 2020 12:38:01 +0300 Subject: [PATCH 3/8] Change arrow style --- mitmproxy/tools/console/common.py | 4 ++++ mitmproxy/tools/console/flowview.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index cba3a3556..46800fddf 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -104,6 +104,8 @@ if urwid.util.detected_encoding: SYMBOL_UP = u"\u21E7" SYMBOL_DOWN = u"\u21E9" SYMBOL_ELLIPSIS = u"\u2026" + SYMBOL_FROM_CLIENT = u"\u21d2" + SYMBOL_TO_CLIENT = u"\u21d0" else: SYMBOL_REPLAY = u"[r]" SYMBOL_RETURN = u"<-" @@ -111,6 +113,8 @@ else: SYMBOL_UP = "^" SYMBOL_DOWN = " " SYMBOL_ELLIPSIS = "~" + SYMBOL_FROM_CLIENT = u"->" + SYMBOL_TO_CLIENT = u"<-" SCHEME_STYLES = { 'http': 'scheme_http', diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 7c6258cfe..bed0a2901 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -144,9 +144,9 @@ class FlowDetails(tabs.Tabs): for line in lines: if message["from_client"]: - line.insert(0, "--> ") + line.insert(0, f"{common.SYMBOL_FROM_CLIENT} ") else: - line.insert(0, "<-- ") + line.insert(0, f"{common.SYMBOL_TO_CLIENT} ") widget_lines.append(urwid.Text(line)) From 607d86965e01fa8972947027e2811cad55331efb Mon Sep 17 00:00:00 2001 From: Nikita Stupin <18281368+nikitastupin@users.noreply.github.com> Date: Sat, 2 May 2020 12:49:02 +0300 Subject: [PATCH 4/8] Colorize arrows! --- mitmproxy/tools/console/flowview.py | 4 ++-- mitmproxy/tools/console/palettes.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index bed0a2901..a4d147440 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -144,9 +144,9 @@ class FlowDetails(tabs.Tabs): for line in lines: if message["from_client"]: - line.insert(0, f"{common.SYMBOL_FROM_CLIENT} ") + line.insert(0, ("from_client", f"{common.SYMBOL_FROM_CLIENT} ")) else: - line.insert(0, f"{common.SYMBOL_TO_CLIENT} ") + line.insert(0, ("to_client", f"{common.SYMBOL_TO_CLIENT} ")) widget_lines.append(urwid.Text(line)) diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index a680a5a7d..15745df59 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -37,6 +37,9 @@ class Palette: # JSON view 'json_string', 'json_number', 'json_boolean', + # TCP flow details + 'from_client', 'to_client', + # Grid Editor 'focusfield', 'focusfield_error', 'field_error', 'editfield', @@ -177,6 +180,10 @@ class LowDark(Palette): json_number = ('light magenta', 'default'), json_boolean = ('dark magenta', 'default'), + # TCP flow details + from_client = ('light blue', 'default'), + to_client = ('light red', 'default'), + # Grid Editor focusfield = ('black', 'light gray'), focusfield_error = ('dark red', 'light gray'), @@ -282,6 +289,10 @@ class LowLight(Palette): json_number = ('light magenta', 'default'), json_boolean = ('dark magenta', 'default'), + # TCP flow details + from_client = ('dark blue', 'default'), + to_client = ('dark red', 'default'), + # Grid Editor focusfield = ('black', 'light gray'), focusfield_error = ('dark red', 'light gray'), @@ -397,6 +408,10 @@ class SolarizedLight(LowLight): json_number = (sol_blue, 'default'), json_boolean = (sol_magenta, 'default'), + # TCP flow details + from_client = (sol_blue, 'default'), + to_client = (sol_red, 'default'), + # Grid Editor focusfield = (sol_base00, sol_base2), focusfield_error = (sol_red, sol_base2), @@ -475,6 +490,10 @@ class SolarizedDark(LowDark): json_number = (sol_blue, 'default'), json_boolean = (sol_magenta, 'default'), + # TCP flow details + from_client = (sol_blue, 'default'), + to_client = (sol_red, 'default'), + # Grid Editor focusfield = (sol_base0, sol_base02), focusfield_error = (sol_red, sol_base02), From 46eb4a06245dfbb86c45f398c47ce0104d0266aa Mon Sep 17 00:00:00 2001 From: Nikita Stupin <18281368+nikitastupin@users.noreply.github.com> Date: Sat, 2 May 2020 13:00:24 +0300 Subject: [PATCH 5/8] Add contentview status bar to TCP flow details --- mitmproxy/tools/console/flowview.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index a4d147440..ff5266df3 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -108,6 +108,26 @@ class FlowDetails(tabs.Tabs): assert isinstance(flow, http.HTTPFlow) return self.conn_text(flow.response) + def _contentview_status_bar(self, description: str, viewmode: str): + cols = [ + urwid.Text( + [ + ("heading", description), + ] + ), + urwid.Text( + [ + " ", + ('heading', "["), + ('heading_key', "m"), + ('heading', (":%s]" % viewmode)), + ], + align="right" + ) + ] + contentview_status_bar = urwid.AttrWrap(urwid.Columns(cols), "heading") + return contentview_status_bar + def view_tcp_stream(self) -> urwid.Widget: flow = self.flow assert isinstance(flow, tcp.TCPFlow) @@ -139,6 +159,9 @@ class FlowDetails(tabs.Tabs): widget_lines = [] + + widget_lines.append(self._contentview_status_bar(viewmode.capitalize(), viewmode)) + for message in merged_messages: _, lines, _ = contentviews.get_tcp_content_view(viewmode, message["content"]) From 3d46e9817171f3b5c6b057c217898479a8c92434 Mon Sep 17 00:00:00 2001 From: Nikita Stupin <18281368+nikitastupin@users.noreply.github.com> Date: Sat, 2 May 2020 13:01:50 +0300 Subject: [PATCH 6/8] Remove blank lines --- mitmproxy/tools/console/flowview.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index ff5266df3..e65743382 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -157,7 +157,6 @@ class FlowDetails(tabs.Tabs): "from_client": message.from_client, }) - widget_lines = [] widget_lines.append(self._contentview_status_bar(viewmode.capitalize(), viewmode)) From 5ce87b72537ae9b6dca73159549444bd4f28e186 Mon Sep 17 00:00:00 2001 From: Nikita Stupin <18281368+nikitastupin@users.noreply.github.com> Date: Sun, 3 May 2020 10:28:43 +0300 Subject: [PATCH 7/8] Assert viewmode not None --- mitmproxy/contentviews/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 063b8cfe1..f5c31eef6 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -121,6 +121,9 @@ def get_tcp_content_view(viewname: str, data: bytes): if not viewmode: viewmode = get("auto") + # https://github.com/mitmproxy/mitmproxy/pull/3970#issuecomment-623024447 + assert viewmode + description, lines, error = get_content_view(viewmode, data) return description, lines, error From fe75f14ea29d0654c3a688ecc4c3669a946ea2a8 Mon Sep 17 00:00:00 2001 From: Nikita Stupin <18281368+nikitastupin@users.noreply.github.com> Date: Sun, 3 May 2020 10:42:31 +0300 Subject: [PATCH 8/8] Revert view_tcp_stream to List[bytes] implementation --- mitmproxy/tools/console/flowview.py | 34 ++++++++++++----------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index e65743382..466ff0c25 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -139,39 +139,33 @@ class FlowDetails(tabs.Tabs): # Merge adjacent TCP "messages". For detailed explanation of this code block see: # https://github.com/mitmproxy/mitmproxy/pull/3970/files/469bd32582f764f9a29607efa4f5b04bd87961fb#r418670880 - merged_messages = [] + from_client = None + messages = [] for message in flow.messages: - - if not merged_messages: - merged_messages.append({ - "content": message.content, - "from_client": message.from_client, - }) - continue - - if merged_messages[-1]["from_client"] == message.from_client: - merged_messages[-1]["content"] += message.content + if message.from_client is not from_client: + messages.append(message.content) + from_client = message.from_client else: - merged_messages.append({ - "content": message.content, - "from_client": message.from_client, - }) + messages[-1] += message.content widget_lines = [] - widget_lines.append(self._contentview_status_bar(viewmode.capitalize(), viewmode)) - - for message in merged_messages: - _, lines, _ = contentviews.get_tcp_content_view(viewmode, message["content"]) + from_client = flow.messages[0].from_client + for m in messages: + _, lines, _ = contentviews.get_tcp_content_view(viewmode, m) for line in lines: - if message["from_client"]: + if from_client: line.insert(0, ("from_client", f"{common.SYMBOL_FROM_CLIENT} ")) else: line.insert(0, ("to_client", f"{common.SYMBOL_TO_CLIENT} ")) widget_lines.append(urwid.Text(line)) + from_client = not from_client + + widget_lines.insert(0, self._contentview_status_bar(viewmode.capitalize(), viewmode)) + return searchable.Searchable(widget_lines) def view_details(self):