Merge pull request #2386 from cortesi/help

console help

mitmproxy/tools/console/commands.py

@@ -1,30 +1,12 @@
 import urwid
 import blinker
 import textwrap
-from mitmproxy.tools.console import common
+from mitmproxy.tools.console import layoutwidget
 from mitmproxy.tools.console import signals
 
 HELP_HEIGHT = 5
 
 
-footer = [
-    ('heading_key', "enter"), ":edit ",
-    ('heading_key', "?"), ":help ",
-]
-
-
-def _mkhelp():
-    text = []
-    keys = [
-        ("enter", "execute command"),
-    ]
-    text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
-    return text
-
-
-help_context = _mkhelp()
-
-
 def fcol(s, width, attr):
     s = str(s)
     return (
@@ -151,7 +133,7 @@ class CommandHelp(urwid.Frame):
         self.set_body(self.widget(txt))
 
 
-class Commands(urwid.Pile):
+class Commands(urwid.Pile, layoutwidget.LayoutWidget):
     title = "Commands"
     keyctx = "commands"
 

mitmproxy/tools/console/eventlog.py

@@ -1,5 +1,6 @@
 import urwid
 from mitmproxy.tools.console import signals
+from mitmproxy.tools.console import layoutwidget
 
 EVENTLOG_SIZE = 10000
 
@@ -8,7 +9,7 @@ class LogBufferWalker(urwid.SimpleListWalker):
     pass
 
 
-class EventLog(urwid.ListBox):
+class EventLog(urwid.ListBox, layoutwidget.LayoutWidget):
     keyctx = "eventlog"
     title = "Events"
 

mitmproxy/tools/console/flowlist.py

@@ -1,52 +1,10 @@
 import urwid
 
 from mitmproxy.tools.console import common
+from mitmproxy.tools.console import layoutwidget
 import mitmproxy.tools.console.master # noqa
 
 
-def _mkhelp():
-    text = []
-    keys = [
-        ("A", "accept all intercepted flows"),
-        ("a", "accept this intercepted flow"),
-        ("b", "save request/response body"),
-        ("C", "export flow to clipboard"),
-        ("d", "delete flow"),
-        ("D", "duplicate flow"),
-        ("e", "toggle eventlog"),
-        ("E", "export flow to file"),
-        ("f", "filter view"),
-        ("F", "toggle follow flow list"),
-        ("L", "load saved flows"),
-        ("m", "toggle flow mark"),
-        ("M", "toggle marked flow view"),
-        ("n", "create a new request"),
-        ("o", "set flow order"),
-        ("r", "replay request"),
-        ("S", "server replay request/s"),
-        ("U", "unmark all marked flows"),
-        ("v", "reverse flow order"),
-        ("V", "revert changes to request"),
-        ("w", "save flows "),
-        ("W", "stream flows to file"),
-        ("X", "kill and delete flow, even if it's mid-intercept"),
-        ("z", "clear flow list or eventlog"),
-        ("Z", "clear unmarked flows"),
-        ("tab", "tab between eventlog and flow list"),
-        ("enter", "view flow"),
-        ("|", "run script on this flow"),
-    ]
-    text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
-    return text
-
-
-help_context = _mkhelp()
-
-footer = [
-    ('heading_key', "?"), ":help ",
-]
-
-
 class FlowItem(urwid.WidgetWrap):
 
     def __init__(self, master, flow):
@@ -109,7 +67,7 @@ class FlowListWalker(urwid.ListWalker):
         return f, pos
 
 
-class FlowListBox(urwid.ListBox):
+class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget):
     title = "Flows"
     keyctx = "flowlist"
 

mitmproxy/tools/console/flowview.py

@@ -8,6 +8,7 @@ import urwid
 from mitmproxy import contentviews
 from mitmproxy import http
 from mitmproxy.tools.console import common
+from mitmproxy.tools.console import layoutwidget
 from mitmproxy.tools.console import flowdetailview
 from mitmproxy.tools.console import searchable
 from mitmproxy.tools.console import signals
@@ -19,82 +20,6 @@ class SearchError(Exception):
     pass
 
 
-def _mkhelp():
-    text = []
-    keys = [
-        ("A", "accept all intercepted flows"),
-        ("a", "accept this intercepted flow"),
-        ("b", "save request/response body"),
-        ("C", "export flow to clipboard"),
-        ("D", "duplicate flow"),
-        ("d", "delete flow"),
-        ("e", "edit request/response"),
-        ("f", "load full body data"),
-        ("m", "change body display mode for this entity\n(default mode can be changed in the options)"),
-        (None,
-         common.highlight_key("automatic", "a") +
-         [("text", ": automatic detection")]
-         ),
-        (None,
-         common.highlight_key("hex", "e") +
-         [("text", ": Hex")]
-         ),
-        (None,
-         common.highlight_key("html", "h") +
-         [("text", ": HTML")]
-         ),
-        (None,
-         common.highlight_key("image", "i") +
-         [("text", ": Image")]
-         ),
-        (None,
-         common.highlight_key("javascript", "j") +
-         [("text", ": JavaScript")]
-         ),
-        (None,
-         common.highlight_key("json", "s") +
-         [("text", ": JSON")]
-         ),
-        (None,
-         common.highlight_key("urlencoded", "u") +
-         [("text", ": URL-encoded data")]
-         ),
-        (None,
-         common.highlight_key("raw", "r") +
-         [("text", ": raw data")]
-         ),
-        (None,
-         common.highlight_key("xml", "x") +
-         [("text", ": XML")]
-         ),
-        ("E", "export flow to file"),
-        ("r", "replay request"),
-        ("V", "revert changes to request"),
-        ("v", "view body in external viewer"),
-        ("w", "save all flows matching current view filter"),
-        ("W", "save this flow"),
-        ("x", "delete body"),
-        ("z", "encode/decode a request/response"),
-        ("tab", "next tab"),
-        ("h, l", "previous tab, next tab"),
-        ("space", "next flow"),
-        ("|", "run script on this flow"),
-        ("/", "search (case sensitive)"),
-        ("n", "repeat search forward"),
-        ("N", "repeat search backwards"),
-    ]
-    text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
-    return text
-
-
-help_context = _mkhelp()
-
-footer = [
-    ('heading_key', "?"), ":help ",
-    ('heading_key', "q"), ":back ",
-]
-
-
 class FlowViewHeader(urwid.WidgetWrap):
 
     def __init__(
@@ -274,7 +199,7 @@ class FlowDetails(tabs.Tabs):
         return self._w.keypress(size, key)
 
 
-class FlowView(urwid.Frame):
+class FlowView(urwid.Frame, layoutwidget.LayoutWidget):
     keyctx = "flowview"
     title = "Flow Details"
 

mitmproxy/tools/console/grideditor/base.py

@@ -1,17 +1,14 @@
 import abc
 import copy
-from typing import Any
+import os
-from typing import Callable
+import typing
-from typing import Container
-from typing import Iterable
-from typing import Optional
-from typing import Sequence
-from typing import Tuple
-from typing import Set # noqa
-
 import urwid
+
+from mitmproxy.utils import strutils
+from mitmproxy import exceptions
 from mitmproxy.tools.console import common
 from mitmproxy.tools.console import signals
+from mitmproxy.tools.console import layoutwidget
 import mitmproxy.tools.console.master # noqa
 
 FOOTER = [
@@ -23,6 +20,21 @@ FOOTER_EDITING = [
 ]
 
 
+def read_file(filename: str, escaped: bool) -> typing.AnyStr:
+    filename = os.path.expanduser(filename)
+    try:
+        with open(filename, "r" if escaped else "rb") as f:
+            d = f.read()
+    except IOError as v:
+        raise exceptions.CommandError(v)
+    if escaped:
+        try:
+            d = strutils.escaped_str_to_bytes(d)
+        except ValueError:
+            raise exceptions.CommandError("Invalid Python-style string encoding.")
+    return d
+
+
 class Cell(urwid.WidgetWrap):
     def get_data(self):
         """
@@ -50,27 +62,28 @@ class Column(metaclass=abc.ABCMeta):
         pass
 
     @abc.abstractmethod
-    def blank(self) -> Any:
+    def blank(self) -> typing.Any:
         pass
 
-    def keypress(self, key: str, editor: "GridEditor") -> Optional[str]:
+    def keypress(self, key: str, editor: "GridEditor") -> typing.Optional[str]:
         return key
 
 
 class GridRow(urwid.WidgetWrap):
+
     def __init__(
             self,
-            focused: Optional[int],
+            focused: typing.Optional[int],
             editing: bool,
             editor: "GridEditor",
-            values: Tuple[Iterable[bytes], Container[int]]
+            values: typing.Tuple[typing.Iterable[bytes], typing.Container[int]]
     ) -> None:
         self.focused = focused
         self.editor = editor
-        self.edit_col = None  # type: Optional[Cell]
+        self.edit_col = None  # type: typing.Optional[Cell]
 
         errors = values[1]
-        self.fields = []  # type: Sequence[Any]
+        self.fields = []  # type: typing.Sequence[typing.Any]
         for i, v in enumerate(values[0]):
             if focused == i and editing:
                 self.edit_col = self.editor.columns[i].Edit(v)
@@ -116,14 +129,14 @@ class GridWalker(urwid.ListWalker):
 
     def __init__(
             self,
-            lst: Iterable[list],
+            lst: typing.Iterable[list],
             editor: "GridEditor"
     ) -> None:
-        self.lst = [(i, set()) for i in lst]  # type: Sequence[Tuple[Any, Set]]
+        self.lst = [(i, set()) for i in lst]  # type: typing.Sequence[typing.Tuple[typing.Any, typing.Set]]
         self.editor = editor
         self.focus = 0
         self.focus_col = 0
-        self.edit_row = None  # type: Optional[GridRow]
+        self.edit_row = None  # type: typing.Optional[GridRow]
 
     def _modified(self):
         self.editor.show_empty_msg()
@@ -253,14 +266,16 @@ FIRST_WIDTH_MIN = 20
 
 
 class BaseGridEditor(urwid.WidgetWrap):
+    title = ""
+    keyctx = "grideditor"
 
     def __init__(
             self,
             master: "mitmproxy.tools.console.master.ConsoleMaster",
             title,
             columns,
-            value: Any,
+            value: typing.Any,
-            callback: Callable[..., None],
+            callback: typing.Callable[..., None],
             *cb_args,
             **cb_kwargs
     ) -> None:
@@ -280,12 +295,8 @@ class BaseGridEditor(urwid.WidgetWrap):
                 first_width = max(len(r), first_width)
         self.first_width = min(first_width, FIRST_WIDTH_MAX)
 
-        title = None
+        h = None
-        if self.title:
+        if any(col.heading for col in self.columns):
-            title = urwid.Text(self.title)
-            title = urwid.Padding(title, align="left", width=("relative", 100))
-            title = urwid.AttrWrap(title, "heading")
-
             headings = []
             for i, col in enumerate(self.columns):
                 c = urwid.Text(col.heading)
@@ -301,15 +312,13 @@ class BaseGridEditor(urwid.WidgetWrap):
 
         self.walker = GridWalker(self.value, self)
         self.lb = GridListBox(self.walker)
-        w = urwid.Frame(
+        w = urwid.Frame(self.lb, header=h)
-            self.lb,
+
-            header=urwid.Pile([title, h]) if title else None
-        )
         super().__init__(w)
         signals.footer_help.send(self, helptext="")
         self.show_empty_msg()
 
-    def view_popping(self):
+    def layout_popping(self):
         res = []
         for i in self.walker.lst:
             if not i[1] and any([x for x in i[0]]):
@@ -323,9 +332,9 @@ class BaseGridEditor(urwid.WidgetWrap):
             self._w.set_footer(
                 urwid.Text(
                     [
-                        ("highlight", "No values. Press "),
+                        ("highlight", "No values - you should add some. Press "),
-                        ("key", "a"),
+                        ("key", "?"),
-                        ("highlight", " to add some."),
+                        ("highlight", " for help."),
                     ]
                 )
             )
@@ -355,31 +364,23 @@ class BaseGridEditor(urwid.WidgetWrap):
             self.walker.left()
         elif key == "right":
             self.walker.right()
-        elif key == "tab":
-            self.walker.tab_next()
-        elif key == "a":
-            self.walker.add()
-        elif key == "A":
-            self.walker.insert()
-        elif key == "d":
-            self.walker.delete_focus()
         elif column.keypress(key, self) and not self.handle_key(key):
             return self._w.keypress(size, key)
 
-    def data_out(self, data: Sequence[list]) -> Any:
+    def data_out(self, data: typing.Sequence[list]) -> typing.Any:
         """
             Called on raw list data, before data is returned through the
             callback.
         """
         return data
 
-    def data_in(self, data: Any) -> Iterable[list]:
+    def data_in(self, data: typing.Any) -> typing.Iterable[list]:
         """
             Called to prepare provided data.
         """
         return data
 
-    def is_error(self, col: int, val: Any) -> Optional[str]:
+    def is_error(self, col: int, val: typing.Any) -> typing.Optional[str]:
         """
             Return None, or a string error message.
         """
@@ -417,31 +418,57 @@ class BaseGridEditor(urwid.WidgetWrap):
         )
         return text
 
+    def cmd_next(self):
+        self.walker.tab_next()
 
-class GridEditor(urwid.WidgetWrap):
+    def cmd_add(self):
+        self.walker.add()
+
+    def cmd_insert(self):
+        self.walker.insert()
+
+    def cmd_delete(self):
+        self.walker.delete_focus()
+
+    def cmd_read_file(self, path):
+        self.walker.set_current_value(read_file(path, False))
+
+    def cmd_read_file_escaped(self, path):
+        self.walker.set_current_value(read_file(path, True))
+
+    def cmd_spawn_editor(self):
+        o = self.walker.get_current_value()
+        if o is not None:
+            n = self.master.spawn_editor(o)
+            n = strutils.clean_hanging_newline(n)
+            self.walker.set_current_value(n)
+
+
+class GridEditor(BaseGridEditor):
     title = None  # type: str
-    columns = None  # type: Sequence[Column]
+    columns = None  # type: typing.Sequence[Column]
+    keyctx = "grideditor"
 
     def __init__(
             self,
             master: "mitmproxy.tools.console.master.ConsoleMaster",
-            value: Any,
+            value: typing.Any,
-            callback: Callable[..., None],
+            callback: typing.Callable[..., None],
             *cb_args,
             **cb_kwargs
     ) -> None:
         super().__init__(
             master,
-            value,
             self.title,
             self.columns,
+            value,
             callback,
             *cb_args,
             **cb_kwargs
         )
 
 
-class FocusEditor(urwid.WidgetWrap):
+class FocusEditor(urwid.WidgetWrap, layoutwidget.LayoutWidget):
     """
         A specialised GridEditor that edits the current focused flow.
     """
@@ -451,27 +478,11 @@ class FocusEditor(urwid.WidgetWrap):
         self.master = master
         self.focus_changed()
 
-    def focus_changed(self):
-        if self.master.view.focus.flow:
-            self._w = BaseGridEditor(
-                self.master.view.focus.flow,
-                self.title,
-                self.columns,
-                self.get_data(self.master.view.focus.flow),
-                self.set_data_update,
-                self.master.view.focus.flow,
-            )
-        else:
-            self._w = urwid.Pile([])
-
     def call(self, v, name, *args, **kwargs):
         f = getattr(v, name, None)
         if f:
             f(*args, **kwargs)
 
-    def view_popping(self):
-        self.call(self._w, "view_popping")
-
     def get_data(self, flow):
         """
             Retrieve the data to edit from the current flow.
@@ -487,3 +498,22 @@ class FocusEditor(urwid.WidgetWrap):
     def set_data_update(self, vals, flow):
         self.set_data(vals, flow)
         signals.flow_change.send(self, flow = flow)
+
+    def key_responder(self):
+        return self._w
+
+    def layout_popping(self):
+        self.call(self._w, "layout_popping")
+
+    def focus_changed(self):
+        if self.master.view.focus.flow:
+            self._w = BaseGridEditor(
+                self.master,
+                self.title,
+                self.columns,
+                self.get_data(self.master.view.focus.flow),
+                self.set_data_update,
+                self.master.view.focus.flow,
+            )
+        else:
+            self._w = urwid.Pile([])

mitmproxy/tools/console/grideditor/col_bytes.py

@@ -1,34 +1,9 @@
-import os
-from typing import Callable, Optional
-
 import urwid
 from mitmproxy.tools.console import signals
 from mitmproxy.tools.console.grideditor import base
 from mitmproxy.utils import strutils
 
 
-def read_file(filename: str, callback: Callable[..., None], escaped: bool) -> Optional[str]:
-    if not filename:
-        return None
-
-    filename = os.path.expanduser(filename)
-    try:
-        with open(filename, "r" if escaped else "rb") as f:
-            d = f.read()
-    except IOError as v:
-        return str(v)
-
-    if escaped:
-        try:
-            d = strutils.escaped_str_to_bytes(d)
-        except ValueError:
-            return "Invalid Python-style string encoding."
-    # TODO: Refactor the status_prompt_path signal so that we
-    # can raise exceptions here and return the content instead.
-    callback(d)
-    return None
-
-
 class Column(base.Column):
     def Display(self, data):
         return Display(data)
@@ -40,29 +15,7 @@ class Column(base.Column):
         return b""
 
     def keypress(self, key, editor):
-        if key == "r":
+        if key in ["enter"]:
-            if editor.walker.get_current_value() is not None:
-                signals.status_prompt_path.send(
-                    self,
-                    prompt="Read file",
-                    callback=read_file,
-                    args=(editor.walker.set_current_value, True)
-                )
-        elif key == "R":
-            if editor.walker.get_current_value() is not None:
-                signals.status_prompt_path.send(
-                    self,
-                    prompt="Read unescaped file",
-                    callback=read_file,
-                    args=(editor.walker.set_current_value, False)
-                )
-        elif key == "e":
-            o = editor.walker.get_current_value()
-            if o is not None:
-                n = editor.master.spawn_editor(o)
-                n = strutils.clean_hanging_newline(n)
-                editor.walker.set_current_value(n)
-        elif key in ["enter"]:
             editor.walker.start_edit()
         else:
             return key

mitmproxy/tools/console/grideditor/editors.py

@@ -1,17 +1,14 @@
-import re
 
 import urwid
 
 from mitmproxy import exceptions
-from mitmproxy import flowfilter
-from mitmproxy.addons import script
 from mitmproxy.tools.console import common
+from mitmproxy.tools.console import layoutwidget
 from mitmproxy.tools.console.grideditor import base
 from mitmproxy.tools.console.grideditor import col_text
 from mitmproxy.tools.console.grideditor import col_bytes
 from mitmproxy.tools.console.grideditor import col_subgrid
 from mitmproxy.tools.console import signals
-from mitmproxy.net.http import user_agents
 from mitmproxy.net.http import Headers
 
 
@@ -41,7 +38,6 @@ class HeaderEditor(base.FocusEditor):
             urwid.Text([("text", "Special keys:\n")])
         ]
         keys = [
-            ("U", "add User-Agent header"),
         ]
         text.extend(
             common.format_keyvals(keys, key="key", val="text", indent=4)
@@ -50,25 +46,6 @@ class HeaderEditor(base.FocusEditor):
         text.extend(h)
         return text
 
-    def set_user_agent(self, k):
-        ua = user_agents.get_by_shortcut(k)
-        if ua:
-            self.walker.add_value(
-                [
-                    b"User-Agent",
-                    ua[2].encode()
-                ]
-            )
-
-    def handle_key(self, key):
-        if key == "U":
-            signals.status_prompt_onekey.send(
-                prompt="Add User-Agent header:",
-                keys=[(i[0], i[1]) for i in user_agents.UASTRINGS],
-                callback=self.set_user_agent,
-            )
-            return True
-
 
 class RequestHeaderEditor(HeaderEditor):
     title = "Edit Request Headers"
@@ -104,56 +81,6 @@ class RequestFormEditor(base.FocusEditor):
         flow.request.urlencoded_form = vals
 
 
-class SetHeadersEditor(base.GridEditor):
-    title = "Editing header set patterns"
-    columns = [
-        col_text.Column("Filter"),
-        col_text.Column("Header"),
-        col_text.Column("Value"),
-    ]
-
-    def is_error(self, col, val):
-        if col == 0:
-            if not flowfilter.parse(val):
-                return "Invalid filter specification"
-        return False
-
-    def make_help(self):
-        h = super().make_help()
-        text = [
-            urwid.Text([("text", "Special keys:\n")])
-        ]
-        keys = [
-            ("U", "add User-Agent header"),
-        ]
-        text.extend(
-            common.format_keyvals(keys, key="key", val="text", indent=4)
-        )
-        text.append(urwid.Text([("text", "\n")]))
-        text.extend(h)
-        return text
-
-    def set_user_agent(self, k):
-        ua = user_agents.get_by_shortcut(k)
-        if ua:
-            self.walker.add_value(
-                [
-                    ".*",
-                    b"User-Agent",
-                    ua[2].encode()
-                ]
-            )
-
-    def handle_key(self, key):
-        if key == "U":
-            signals.status_prompt_onekey.send(
-                prompt="Add User-Agent header:",
-                keys=[(i[0], i[1]) for i in user_agents.UASTRINGS],
-                callback=self.set_user_agent,
-            )
-            return True
-
-
 class PathEditor(base.FocusEditor):
     # TODO: Next row on enter?
 
@@ -175,38 +102,6 @@ class PathEditor(base.FocusEditor):
         flow.request.path_components = self.data_out(vals)
 
 
-class ScriptEditor(base.GridEditor):
-    title = "Editing scripts"
-    columns = [
-        col_text.Column("Command"),
-    ]
-
-    def is_error(self, col, val):
-        try:
-            script.parse_command(val)
-        except exceptions.OptionsError as e:
-            return str(e)
-
-
-class HostPatternEditor(base.GridEditor):
-    title = "Editing host patterns"
-    columns = [
-        col_text.Column("Regex (matched on hostname:port / ip:port)")
-    ]
-
-    def is_error(self, col, val):
-        try:
-            re.compile(val, re.IGNORECASE)
-        except re.error as e:
-            return "Invalid regex: %s" % str(e)
-
-    def data_in(self, data):
-        return [[i] for i in data]
-
-    def data_out(self, data):
-        return [i[0] for i in data]
-
-
 class CookieEditor(base.FocusEditor):
     title = "Edit Cookies"
     columns = [
@@ -273,7 +168,7 @@ class SetCookieEditor(base.FocusEditor):
         flow.response.cookies = self.data_out(vals)
 
 
-class OptionsEditor(base.GridEditor):
+class OptionsEditor(base.GridEditor, layoutwidget.LayoutWidget):
     title = None  # type: str
     columns = [
         col_text.Column("")

mitmproxy/tools/console/help.py

@@ -1,66 +1,67 @@
-import platform
-
 import urwid
 
 from mitmproxy import flowfilter
 from mitmproxy.tools.console import common
-
+from mitmproxy.tools.console import layoutwidget
-from mitmproxy import version
+from mitmproxy.tools.console import tabs
-
-footer = [
-    ("heading", 'mitmproxy {} (Python {}) '.format(version.VERSION, platform.python_version())),
-    ('heading_key', "q"), ":back ",
-]
 
 
-class HelpView(urwid.ListBox):
+class HelpView(tabs.Tabs, layoutwidget.LayoutWidget):
     title = "Help"
     keyctx = "help"
 
-    def __init__(self, help_context):
+    def __init__(self, master):
-        self.help_context = help_context or []
+        self.master = master
-        urwid.ListBox.__init__(
+        self.helpctx = ""
-            self,
+        super().__init__(
-            self.helptext()
+            [
+                [self.keybindings_title, self.keybindings],
+                [self.filtexp_title, self.filtexp],
+            ]
         )
 
-    def helptext(self):
+    def keybindings_title(self):
+        return "Key Bindings"
+
+    def format_keys(self, binds):
+        kvs = []
+        for b in binds:
+            k = b.key
+            if b.key == " ":
+                k = "space"
+            kvs.append((k, b.command))
+        return common.format_keyvals(kvs)
+
+    def keybindings(self):
+        text = [
+            urwid.Text(
+                [
+                    ("title", "Keybindings for this view")
+                ]
+            )
+        ]
+        if self.helpctx:
+            text.extend(self.format_keys(self.master.keymap.list(self.helpctx)))
+
+        text.append(
+            urwid.Text(
+                [
+                    "\n",
+                    ("title", "Global Keybindings"),
+                ]
+            )
+        )
+
+        text.extend(self.format_keys(self.master.keymap.list("global")))
+
+        return urwid.ListBox(text)
+
+    def filtexp_title(self):
+        return "Filter Expressions"
+
+    def filtexp(self):
         text = []
-        text.append(urwid.Text([("head", "This view:\n")]))
-        text.extend(self.help_context)
-
-        text.append(urwid.Text([("head", "\n\nMovement:\n")]))
-        keys = [
-            ("j, k", "down, up"),
-            ("h, l", "left, right (in some contexts)"),
-            ("g, G", "go to beginning, end"),
-            ("space", "page down"),
-            ("pg up/down", "page up/down"),
-            ("ctrl+b/ctrl+f", "page up/down"),
-            ("arrows", "up, down, left, right"),
-        ]
-        text.extend(
-            common.format_keyvals(
-                keys,
-                key="key",
-                val="text",
-                indent=4))
-
-        text.append(urwid.Text([("head", "\n\nGlobal keys:\n")]))
-        keys = [
-            ("i", "set interception pattern"),
-            ("O", "options"),
-            ("q", "quit / return to previous page"),
-            ("Q", "quit without confirm prompt"),
-            ("R", "replay of requests/responses from file"),
-        ]
-        text.extend(
-            common.format_keyvals(keys, key="key", val="text", indent=4)
-        )
-
-        text.append(urwid.Text([("head", "\n\nFilter expressions:\n")]))
         text.extend(common.format_keyvals(flowfilter.help, key="key", val="text", indent=4))
-
         text.append(
             urwid.Text(
                 [
@@ -82,11 +83,11 @@ class HelpView(urwid.ListBox):
         text.extend(
             common.format_keyvals(examples, key="key", val="text", indent=4)
         )
-        return text
+        return urwid.ListBox(text)
 
-    def keypress(self, size, key):
+    def layout_pushed(self, prev):
-        if key == "m_start":
+        """
-            self.set_focus(0)
+            We are just about to push a window onto the stack.
-        elif key == "m_end":
+        """
-            self.set_focus(len(self.body.contents))
+        self.helpctx = prev.keyctx
-        return urwid.ListBox.keypress(self, size, key)
+        self.show()

mitmproxy/tools/console/keymap.py

@@ -49,6 +49,11 @@ class Keymap:
             return self.keys[context].get(key, None)
         return None
 
+    def list(self, context: str) -> typing.Sequence[Binding]:
+        b = [b for b in self.bindings if context in b.contexts]
+        b.sort(key=lambda x: x.key)
+        return b
+
     def handle(self, context: str, key: str) -> typing.Optional[str]:
         """
             Returns the key if it has not been handled, or None.

mitmproxy/tools/console/layoutwidget.py

@@ -0,0 +1,42 @@
+
+
+class LayoutWidget:
+    """
+        All top-level layout widgets and all widgets that may be set in an
+        overlay must comply with this API.
+    """
+    # Title is only required for windows, not overlay components
+    title = ""
+    keyctx = ""
+
+    def key_responder(self):
+        """
+            Returns the object responding to key input. Usually self, but may be
+            a wrapped object.
+        """
+        return self
+
+    def focus_changed(self):
+        """
+            The view focus has changed. Layout objects should implement the API
+            rather than directly subscribing to events.
+        """
+        pass
+
+    def view_changed(self):
+        """
+            The view list has changed.
+        """
+        pass
+
+    def layout_popping(self):
+        """
+            We are just about to pop a window off the stack, or exit an overlay.
+        """
+        pass
+
+    def layout_pushed(self, prev):
+        """
+            We have just pushed a window onto the stack.
+        """
+        pass

mitmproxy/tools/console/master.py

@@ -322,6 +322,62 @@ class ConsoleAddon:
                 "console.command flow.set @focus %s " % part
             )
 
+    def _grideditor(self):
+        gewidget = self.master.window.current("grideditor")
+        if not gewidget:
+            raise exceptions.CommandError("Not in a grideditor.")
+        return gewidget.key_responder()
+
+    @command.command("console.grideditor.add")
+    def grideditor_add(self) -> None:
+        """
+            Add a row after the cursor.
+        """
+        self._grideditor().cmd_add()
+
+    @command.command("console.grideditor.insert")
+    def grideditor_insert(self) -> None:
+        """
+            Insert a row before the cursor.
+        """
+        self._grideditor().cmd_insert()
+
+    @command.command("console.grideditor.next")
+    def grideditor_next(self) -> None:
+        """
+            Go to next cell.
+        """
+        self._grideditor().cmd_next()
+
+    @command.command("console.grideditor.delete")
+    def grideditor_delete(self) -> None:
+        """
+            Delete row
+        """
+        self._grideditor().cmd_delete()
+
+    @command.command("console.grideditor.readfile")
+    def grideditor_readfile(self, path: str) -> None:
+        """
+            Read a file into the currrent cell.
+        """
+        self._grideditor().cmd_read_file(path)
+
+    @command.command("console.grideditor.readfile_escaped")
+    def grideditor_readfile_escaped(self, path: str) -> None:
+        """
+            Read a file containing a Python-style escaped stringinto the
+            currrent cell.
+        """
+        self._grideditor().cmd_read_file_escaped(path)
+
+    @command.command("console.grideditor.editor")
+    def grideditor_editor(self) -> None:
+        """
+            Spawn an external editor on the current cell.
+        """
+        self._grideditor().cmd_spawn_editor()
+
     @command.command("console.flowview.mode.set")
     def flowview_mode_set(self) -> None:
         """
@@ -349,7 +405,7 @@ class ConsoleAddon:
         """
             Get the display mode for the current flow view.
         """
-        fv = self.master.window.any("flowview")
+        fv = self.master.window.current_window("flowview")
         if not fv:
             raise exceptions.CommandError("Not viewing a flow.")
         idx = fv.body.tab_offset
@@ -476,6 +532,14 @@ def default_keymap(km):
     km.add("D", "options.reset", ["options"])
     km.add("d", "console.options.reset.current", ["options"])
 
+    km.add("a", "console.grideditor.add", ["grideditor"])
+    km.add("A", "console.grideditor.insert", ["grideditor"])
+    km.add("tab", "console.grideditor.next", ["grideditor"])
+    km.add("d", "console.grideditor.delete", ["grideditor"])
+    km.add("r", "console.command console.grideditor.readfile", ["grideditor"])
+    km.add("R", "console.command console.grideditor.readfile_escaped", ["grideditor"])
+    km.add("e", "console.grideditor.editor", ["grideditor"])
+
 
 class ConsoleMaster(master.Master):
 

mitmproxy/tools/console/options.py

@@ -6,7 +6,7 @@ from typing import Optional, Sequence
 
 from mitmproxy import exceptions
 from mitmproxy import optmanager
-from mitmproxy.tools.console import common
+from mitmproxy.tools.console import layoutwidget
 from mitmproxy.tools.console import signals
 from mitmproxy.tools.console import overlay
 
@@ -20,28 +20,6 @@ def can_edit_inplace(opt):
         return True
 
 
-footer = [
-    ('heading_key', "enter"), ":edit ",
-    ('heading_key', "?"), ":help ",
-]
-
-
-def _mkhelp():
-    text = []
-    keys = [
-        ("enter", "edit option"),
-        ("D", "reset all to defaults"),
-        ("d", "reset this option to default"),
-        ("l", "load options from file"),
-        ("w", "save options to file"),
-    ]
-    text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
-    return text
-
-
-help_context = _mkhelp()
-
-
 def fcol(s, width, attr):
     s = str(s)
     return (
@@ -263,7 +241,7 @@ class OptionHelp(urwid.Frame):
         self.set_body(self.widget(txt))
 
 
-class Options(urwid.Pile):
+class Options(urwid.Pile, layoutwidget.LayoutWidget):
     title = "Options"
     keyctx = "options"
 

mitmproxy/tools/console/overlay.py

@@ -5,10 +5,10 @@ import urwid
 from mitmproxy.tools.console import common
 from mitmproxy.tools.console import signals
 from mitmproxy.tools.console import grideditor
+from mitmproxy.tools.console import layoutwidget
 
 
-class SimpleOverlay(urwid.Overlay):
+class SimpleOverlay(urwid.Overlay, layoutwidget.LayoutWidget):
-    keyctx = "overlay"
 
     def __init__(self, master, widget, parent, width, valign="middle"):
         self.widget = widget
@@ -22,14 +22,21 @@ class SimpleOverlay(urwid.Overlay):
             height="pack"
         )
 
-    def keypress(self, size, key):
+    @property
-        key = super().keypress(size, key)
+    def keyctx(self):
-        if key == "esc":
+        return getattr(self.widget, "keyctx")
-            signals.pop_view_state.send(self)
+
-        if key == "?":
+    def key_responder(self):
-            self.master.view_help(self.widget.make_help())
+        return self.widget.key_responder()
-        else:
+
-            return key
+    def focus_changed(self):
+        return self.widget.focus_changed()
+
+    def view_changed(self):
+        return self.widget.view_changed()
+
+    def layout_popping(self):
+        return self.widget.layout_popping()
 
 
 class Choice(urwid.WidgetWrap):
@@ -81,7 +88,9 @@ class ChooserListWalker(urwid.ListWalker):
         return self._get(pos, False), pos
 
 
-class Chooser(urwid.WidgetWrap):
+class Chooser(urwid.WidgetWrap, layoutwidget.LayoutWidget):
+    keyctx = "chooser"
+
     def __init__(self, master, title, choices, current, callback):
         self.master = master
         self.choices = choices
@@ -122,7 +131,9 @@ class Chooser(urwid.WidgetWrap):
         return text
 
 
-class OptionsOverlay(urwid.WidgetWrap):
+class OptionsOverlay(urwid.WidgetWrap, layoutwidget.LayoutWidget):
+    keyctx = "grideditor"
+
     def __init__(self, master, name, vals, vspace):
         """
             vspace: how much vertical space to keep clear
@@ -142,3 +153,9 @@ class OptionsOverlay(urwid.WidgetWrap):
 
     def make_help(self):
         return self.ge.make_help()
+
+    def key_responder(self):
+        return self.ge.key_responder()
+
+    def layout_popping(self):
+        return self.ge.layout_popping()

mitmproxy/tools/console/statusbar.py

@@ -146,24 +146,18 @@ class StatusBar(urwid.WidgetWrap):
     keyctx = ""
 
     def __init__(
-        self, master: "mitmproxy.tools.console.master.ConsoleMaster", helptext
+        self, master: "mitmproxy.tools.console.master.ConsoleMaster"
     ) -> None:
         self.master = master
-        self.helptext = helptext
         self.ib = urwid.WidgetWrap(urwid.Text(""))
         self.ab = ActionBar(self.master)
         super().__init__(urwid.Pile([self.ib, self.ab]))
         signals.update_settings.connect(self.sig_update)
         signals.flowlist_change.connect(self.sig_update)
-        signals.footer_help.connect(self.sig_footer_help)
         master.options.changed.connect(self.sig_update)
         master.view.focus.sig_change.connect(self.sig_update)
         self.redraw()
 
-    def sig_footer_help(self, sender, helptext):
-        self.helptext = helptext
-        self.redraw()
-
     def sig_update(self, sender, updated=None):
         self.redraw()
 
@@ -288,13 +282,7 @@ class StatusBar(urwid.WidgetWrap):
         t.extend(self.get_status())
         status = urwid.AttrWrap(urwid.Columns([
             urwid.Text(t),
-            urwid.Text(
+            urwid.Text(boundaddr, align="right"),
-                [
-                    self.helptext,
-                    boundaddr
-                ],
-                align="right"
-            ),
         ]), "heading")
         self.ib._w = status
 

mitmproxy/tools/console/window.py

@@ -30,7 +30,7 @@ class WindowStack:
             flowview = flowview.FlowView(master),
             commands = commands.Commands(master),
             options = options.Options(master),
-            help = help.HelpView(None),
+            help = help.HelpView(master),
             eventlog = eventlog.EventLog(master),
 
             edit_focus_query = grideditor.QueryEditor(master),
@@ -45,43 +45,57 @@ class WindowStack:
         self.overlay = None
 
     def set_overlay(self, o, **kwargs):
-        self.overlay = overlay.SimpleOverlay(self, o, self.top(), o.width, **kwargs)
+        self.overlay = overlay.SimpleOverlay(
+            self, o, self.top_widget(), o.width, **kwargs,
+        )
 
-    @property
+    def top_window(self):
-    def topwin(self):
+        """
+            The current top window, ignoring overlays.
+        """
         return self.windows[self.stack[-1]]
 
-    def top(self):
+    def top_widget(self):
+        """
+            The current top widget - either a window or the active overlay.
+        """
         if self.overlay:
             return self.overlay
-        return self.topwin
+        return self.top_window()
 
     def push(self, wname):
         if self.stack[-1] == wname:
             return
+        prev = self.top_window()
         self.stack.append(wname)
+        self.call("layout_pushed", prev)
 
     def pop(self, *args, **kwargs):
         """
             Pop off the stack, return True if we're already at the top.
         """
+        if not self.overlay and len(self.stack) == 1:
+            return True
+        self.call("layout_popping")
         if self.overlay:
             self.overlay = None
-        elif len(self.stack) > 1:
-            self.call("view_popping")
-            self.stack.pop()
         else:
-            return True
+            self.stack.pop()
 
     def call(self, name, *args, **kwargs):
-        f = getattr(self.topwin, name, None)
+        """
-        if f:
+            Call a function on both the top window, and the overlay if there is
-            f(*args, **kwargs)
+            one. If the widget has a key_responder, we call the function on the
+            responder instead.
+        """
+        getattr(self.top_window(), name)(*args, **kwargs)
+        if self.overlay:
+            getattr(self.overlay, name)(*args, **kwargs)
 
 
 class Window(urwid.Frame):
     def __init__(self, master):
-        self.statusbar = statusbar.StatusBar(master, "")
+        self.statusbar = statusbar.StatusBar(master)
         super().__init__(
             None,
             header = None,
@@ -122,24 +136,26 @@ class Window(urwid.Frame):
         if c == "single":
             self.pane = 0
 
-        def wrap(w, idx):
+        def wrapped(idx):
-            if self.master.options.console_layout_headers and hasattr(w, "title"):
+            window = self.stacks[idx].top_window()
-                return Header(w, w.title, self.pane == idx)
+            widget = self.stacks[idx].top_widget()
+            if self.master.options.console_layout_headers and window.title:
+                return Header(widget, window.title, self.pane == idx)
             else:
-                return w
+                return widget
 
         w = None
         if c == "single":
-            w = wrap(self.stacks[0].top(), 0)
+            w = wrapped(0)
         elif c == "vertical":
             w = urwid.Pile(
                 [
-                    wrap(s.top(), i) for i, s in enumerate(self.stacks)
+                    wrapped(i) for i, s in enumerate(self.stacks)
                 ]
             )
         else:
             w = urwid.Columns(
-                [wrap(s.top(), i) for i, s in enumerate(self.stacks)],
+                [wrapped(i) for i, s in enumerate(self.stacks)],
                 dividechars=1
             )
 
@@ -195,11 +211,18 @@ class Window(urwid.Frame):
 
     def current(self, keyctx):
         """
-
+            Returns the active widget, but only the current focus or overlay has
-            Returns the top window of the current stack, IF the current focus
+            a matching key context.
-            has a matching key context.
         """
-        t = self.focus_stack().topwin
+        t = self.focus_stack().top_widget()
+        if t.keyctx == keyctx:
+            return t
+
+    def current_window(self, keyctx):
+        """
+            Returns the active window, ignoring overlays.
+        """
+        t = self.focus_stack().top_window()
         if t.keyctx == keyctx:
             return t
 
@@ -207,7 +230,7 @@ class Window(urwid.Frame):
         """
             Returns the top window of either stack if they match the context.
         """
-        for t in [x.topwin for x in self.stacks]:
+        for t in [x.top_window() for x in self.stacks]:
             if t.keyctx == keyctx:
                 return t
 
@@ -245,7 +268,7 @@ class Window(urwid.Frame):
         if self.focus_part == "footer":
             return super().keypress(size, k)
         else:
-            fs = self.focus_stack().top()
+            fs = self.focus_stack().top_widget()
             k = fs.keypress(size, k)
             if k:
                 return self.master.keymap.handle(fs.keyctx, k)

test/mitmproxy/tools/console/test_help.py

@@ -1,11 +0,0 @@
-import mitmproxy.tools.console.help as help
-
-from ....conftest import skip_appveyor
-
-
-@skip_appveyor
-class TestHelp:
-
-    def test_helptext(self):
-        h = help.HelpView(None)
-        assert h.helptext()

test/mitmproxy/tools/console/test_keymap.py

@@ -16,6 +16,7 @@ def test_bind():
         assert km.get("options", "key")
         assert km.get("commands", "key")
         assert not km.get("flowlist", "key")
+        assert len((km.list("commands"))) == 1
 
         km.handle("unknown", "unknown")
         assert not km.executor.called
@@ -27,3 +28,5 @@ def test_bind():
         km.executor = mock.Mock()
         km.handle("options", "glob")
         assert km.executor.called
+
+        assert len((km.list("global"))) == 1

tox.ini

@@ -54,8 +54,8 @@ commands =
 deps =
   -rrequirements.txt
   -e./release
-  # The 3.2 release is broken 🎉
+  # The 3.2 release is broken 
-  # the next commit after this updates the bootloaders, which then segfault! 🎉
+  # the next commit after this updates the bootloaders, which then segfault!
   # https://github.com/pyinstaller/pyinstaller/issues/2232
   git+https://github.com/pyinstaller/pyinstaller.git@483c819d6a256b58db6740696a901bd41c313f0c; sys_platform == 'win32'
   git+https://github.com/mhils/pyinstaller.git@d094401e4196b1a6a03818b80164a5f555861cef; sys_platform != 'win32'