Use emoji and characters as markers. Add ~marker filter syntax. (#4587)

* Use emoji and characters as markers. Add ~marker filter syntax.

* Add a test to please our CI overlords. :)

CHANGELOG.md

@@ -63,6 +63,8 @@ If you depend on these features, please raise your voice in
 * Improve readability of SHA256 fingerprint. (@wrekone)
 * Metadata and Replay Flow Filters: Flows may be filtered based on metadata and replay status. (@rbdixon)
 * Flow control: don't read connection data faster than it can be forwarded. (@hazcod)
+* Customize markers with emoji, and filters: The `flow.mark` command may be used to mark a flow with either the default
+  "red ball" marker, a single character, or an emoji like `:grapes:`. Use the `~marker` filter to filter on marker characters. (@rbdixon)
 * --- TODO: add new PRs above this line ---
 * ... and various other fixes, documentation improvements, dependency version bumps, etc.
 

examples/contrib/all_markers.py

@@ -0,0 +1,10 @@
+from mitmproxy import ctx, command
+from mitmproxy.utils import emoji
+
+
+@command.command('all.markers')
+def all_markers():
+    'Create a new flow showing all marker values'
+    for marker in emoji.emoji:
+        ctx.master.commands.call('view.flows.create', 'get', f'https://example.com/{marker}')
+        ctx.master.commands.call('flow.mark', [ctx.master.view.focus.flow], marker)

mitmproxy/addons/core.py

@@ -2,7 +2,7 @@ import typing
 
 import os
 
-from mitmproxy.utils import human
+from mitmproxy.utils import emoji, human
 from mitmproxy import ctx, hooks
 from mitmproxy import exceptions
 from mitmproxy import command
@@ -103,15 +103,17 @@ class Core:
 
     # FIXME: this will become view.mark later
     @command.command("flow.mark")
-    def mark(self, flows: typing.Sequence[flow.Flow], boolean: bool) -> None:
+    def mark(self, flows: typing.Sequence[flow.Flow], marker: mitmproxy.types.Marker) -> None:
         """
             Mark flows.
         """
         updated = []
+        if marker not in emoji.emoji:
+            raise exceptions.CommandError(f"invalid marker value")
+
         for i in flows:
-            if i.marked != boolean:
-                i.marked = boolean
-                updated.append(i)
+            i.marked = marker
+            updated.append(i)
         ctx.master.addons.trigger(hooks.UpdateHook(updated))
 
     # FIXME: this will become view.mark.toggle later
@@ -121,7 +123,10 @@ class Core:
             Toggle mark for flows.
         """
         for i in flows:
-            i.marked = not i.marked
+            if i.marked:
+                i.marked = ""
+            else:
+                i.marked = ":default:"
         ctx.master.addons.trigger(hooks.UpdateHook(flows))
 
     @command.command("flow.kill")

mitmproxy/flow.py

@@ -82,9 +82,18 @@ class Flow(stateobject.StateObject):
     We're waiting for a user action to forward the flow to its destination.
     """
 
-    marked: bool
+    marked: str = ""
     """
-    If `True`, this flow has been marked by the user.
+    If this attribute is a non-empty string the flow has been marked by the user.
+
+    A string value will be used as the marker annotation. May either be a single character or a Unicode emoji name.
+
+    For example `:grapes:` becomes `🍇` in views that support emoji rendering.
+    Consult the [Github API Emoji List](https://api.github.com/emojis) for a list of emoji that may be used.
+    Not all emoji, especially [emoji modifiers](https://en.wikipedia.org/wiki/Miscellaneous_Symbols_and_Pictographs#Emoji_modifiers)
+    will render consistently.
+
+    The default marker for the view will be used if the Unicode emoji name can not be interpreted.
     """
 
     is_replay: typing.Optional[str]
@@ -111,7 +120,7 @@ class Flow(stateobject.StateObject):
         self.intercepted: bool = False
         self._backup: typing.Optional[Flow] = None
         self.reply: typing.Optional[controller.Reply] = None
-        self.marked: bool = False
+        self.marked: str = ""
         self.is_replay: typing.Optional[str] = None
         self.metadata: typing.Dict[str, typing.Any] = dict()
 
@@ -123,7 +132,7 @@ class Flow(stateobject.StateObject):
         type=str,
         intercepted=bool,
         is_replay=str,
-        marked=bool,
+        marked=str,
         metadata=typing.Dict[str, typing.Any],
     )
 

mitmproxy/flowfilter.py

@@ -86,7 +86,7 @@ class FMarked(_Action):
     help = "Match marked flows"
 
     def __call__(self, f):
-        return f.marked
+        return bool(f.marked)
 
 
 class FHTTP(_Action):
@@ -416,6 +416,15 @@ class FMeta(_Rex):
         return self.re.search(m)
 
 
+class FMarker(_Rex):
+    code = "marker"
+    help = "Match marked flows with specified marker"
+    is_binary = False
+
+    def __call__(self, f):
+        return self.re.search(f.marked)
+
+
 class _Int(_Action):
 
     def __init__(self, num):
@@ -502,6 +511,7 @@ filter_rex: Sequence[Type[_Rex]] = [
     FSrc,
     FUrl,
     FMeta,
+    FMarker,
 ]
 filter_int = [
     FCode

mitmproxy/io/compat.py

@@ -300,6 +300,15 @@ def convert_11_12(data):
     return data
 
 
+def convert_12_13(data):
+    data["version"] = 13
+    if data["marked"]:
+        data["marked"] = ":default:"
+    else:
+        data["marked"] = ""
+    return data
+
+
 def _convert_dict_keys(o: Any) -> Any:
     if isinstance(o, dict):
         return {strutils.always_str(k): _convert_dict_keys(v) for k, v in o.items()}
@@ -359,6 +368,7 @@ converters = {
     9: convert_9_10,
     10: convert_10_11,
     11: convert_11_12,
+    12: convert_12_13,
 }
 
 

mitmproxy/tools/console/common.py

@@ -10,7 +10,7 @@ import urwid.util
 
 from mitmproxy import flow
 from mitmproxy.http import HTTPFlow
-from mitmproxy.utils import human
+from mitmproxy.utils import human, emoji
 from mitmproxy.tcp import TCPFlow
 
 # Detect Windows Subsystem for Linux and Windows
@@ -161,6 +161,16 @@ def fixlen_r(s: str, maxlen: int) -> str:
         return SYMBOL_ELLIPSIS + s[len(s) - maxlen + len(SYMBOL_ELLIPSIS):]
 
 
+def render_marker(marker: str) -> str:
+    rendered = emoji.emoji.get(marker, SYMBOL_MARK)
+
+    # The marker can only be one glyph. Some emoji that use zero-width joiners (ZWJ)
+    # will not be rendered as a single glyph and instead will show
+    # multiple glyphs. Just use the first glyph as a fallback.
+    # https://emojipedia.org/emoji-zwj-sequence/
+    return rendered[0]
+
+
 class TruncatedText(urwid.Widget):
     def __init__(self, text, attr, align='left'):
         self.text = text
@@ -359,18 +369,18 @@ def format_left_indicators(
 def format_right_indicators(
         *,
         replay: bool,
-        marked: bool
+        marked: str,
 ):
     indicators: typing.List[typing.Union[str, typing.Tuple[str, str]]] = []
     if replay:
         indicators.append(("replay", SYMBOL_REPLAY))
     else:
         indicators.append(" ")
-    if marked:
-        indicators.append(("mark", SYMBOL_MARK))
+    if bool(marked):
+        indicators.append(("mark", render_marker(marked)))
     else:
-        indicators.append(" ")
-    return "fixed", 2, urwid.Text(indicators)
+        indicators.append("  ")
+    return "fixed", 3, urwid.Text(indicators)
 
 
 @lru_cache(maxsize=800)
@@ -378,7 +388,7 @@ def format_http_flow_list(
         *,
         render_mode: RenderMode,
         focused: bool,
-        marked: bool,
+        marked: str,
         is_replay: bool,
         request_method: str,
         request_scheme: str,
@@ -479,7 +489,7 @@ def format_http_flow_table(
         *,
         render_mode: RenderMode,
         focused: bool,
-        marked: bool,
+        marked: str,
         is_replay: typing.Optional[str],
         request_method: str,
         request_scheme: str,
@@ -574,7 +584,7 @@ def format_http_flow_table(
 
     items.append(format_right_indicators(
         replay=bool(is_replay),
-        marked=marked
+        marked=marked,
     ))
     return urwid.Columns(items, dividechars=1, min_width=15)
 
@@ -585,7 +595,7 @@ def format_tcp_flow(
         render_mode: RenderMode,
         focused: bool,
         timestamp_start: float,
-        marked: bool,
+        marked: str,
         client_address,
         server_address,
         total_size: int,

mitmproxy/types.py

@@ -4,6 +4,7 @@ import typing
 
 from mitmproxy import exceptions
 from mitmproxy import flow
+from mitmproxy.utils import emoji
 
 if typing.TYPE_CHECKING:  # pragma: no cover
     from mitmproxy.command import CommandManager
@@ -37,6 +38,10 @@ class Data(typing.Sequence[typing.Sequence[typing.Union[str, bytes]]]):
     pass
 
 
+class Marker(str):
+    pass
+
+
 class Choice:
     def __init__(self, options_command):
         self.options_command = options_command
@@ -406,6 +411,29 @@ class _ChoiceType(_BaseType):
         return val in opts
 
 
+ALL_MARKERS = ['true', 'false'] + list(emoji.emoji)
+
+
+class _MarkerType(_BaseType):
+    typ = Marker
+    display = "marker"
+
+    def completion(self, manager: "CommandManager", t: Choice, s: str) -> typing.Sequence[str]:
+        return ALL_MARKERS
+
+    def parse(self, manager: "CommandManager", t: Choice, s: str) -> str:
+        if s not in ALL_MARKERS:
+            raise exceptions.TypeError("Invalid choice.")
+        if s == 'true':
+            return ":default:"
+        elif s == 'false':
+            return ""
+        return s
+
+    def is_valid(self, manager: "CommandManager", typ: typing.Any, val: str) -> bool:
+        return val in ALL_MARKERS
+
+
 class TypeManager:
     def __init__(self, *types):
         self.typemap = {}
@@ -428,6 +456,7 @@ CommandTypes = TypeManager(
     _FlowType,
     _FlowsType,
     _IntType,
+    _MarkerType,
     _PathType,
     _StrType,
     _StrSeqType,

mitmproxy/version.py

@@ -7,7 +7,7 @@ MITMPROXY = "mitmproxy " + VERSION
 
 # Serialization format version. This is displayed nowhere, it just needs to be incremented by one
 # for each change in the file format.
-FLOW_FORMAT_VERSION = 12
+FLOW_FORMAT_VERSION = 13
 
 
 def get_dev_version() -> str:

test/mitmproxy/addons/test_core.py

@@ -33,9 +33,12 @@ def test_mark():
     with taddons.context(loadcore=False):
         f = tflow.tflow()
         assert not f.marked
-        sa.mark([f], True)
+        sa.mark([f], ":default:")
         assert f.marked
 
+        with pytest.raises(exceptions.CommandError):
+            sa.mark([f], "invalid")
+
         sa.mark_toggle([f])
         assert not f.marked
         sa.mark_toggle([f])

test/mitmproxy/addons/test_view.py

@@ -8,6 +8,7 @@ from mitmproxy import exceptions
 from mitmproxy import io
 from mitmproxy.test import taddons
 from mitmproxy.tools.console import consoleaddons
+from mitmproxy.tools.console.common import render_marker, SYMBOL_MARK
 
 
 def tft(*, method="get", start=0):
@@ -614,3 +615,13 @@ def test_configure():
 
         tctx.configure(v, console_focus_follow=True)
         assert v.focus_follow
+
+
+@pytest.mark.parametrize("marker, expected", [
+    [":default:", SYMBOL_MARK],
+    ["X", "X"],
+    [":grapes:", "\N{grapes}"],
+    [":not valid:", SYMBOL_MARK], [":weird", SYMBOL_MARK]
+])
+def test_marker(marker, expected):
+    assert render_marker(marker) == expected

test/mitmproxy/data/dumpfile-10.mitm

@@ -0,0 +1 @@
+1383:7:version;2:10#4:mode;7:regular;8:response;0:~7:request;291:4:path;1:/,9:authority;0:,6:scheme;5:https,6:method;3:GET,4:port;3:443#4:host;11:example.com;13:timestamp_end;17:1621870806.728885^15:timestamp_start;17:1621870806.728884^8:trailers;0:~7:content;0:,7:headers;52:22:14:content-length,1:0,]22:4:Host,11:example.com,]]12:http_version;8:HTTP/1.1,}8:metadata;0:}6:marked;4:true!9:is_replay;0:~11:intercepted;5:false!4:type;4:http;11:server_conn;468:3:via;0:~4:via2;0:~11:cipher_list;0:]11:cipher_name;0:~11:alpn_offers;0:]16:certificate_list;0:]3:tls;5:false!5:error;0:~5:state;1:0#13:timestamp_end;0:~19:timestamp_tls_setup;0:~19:timestamp_tcp_setup;0:~15:timestamp_start;0:~11:tls_version;0:~21:alpn_proto_negotiated;0:~3:sni;11:example.com;15:tls_established;5:false!14:source_address;7:0:;1:0#]10:ip_address;21:11:example.com;3:443#]7:address;21:11:example.com;3:443#]2:id;36:14ddf3ed-32ad-4e46-a049-38e214ecaa7b;}11:client_conn;385:11:cipher_list;0:]11:alpn_offers;0:]16:certificate_list;0:]3:tls;5:false!5:error;0:~8:sockname;7:0:;1:0#]5:state;1:0#14:tls_extensions;0:~11:tls_version;0:~21:alpn_proto_negotiated;0:~11:cipher_name;0:~3:sni;0:~13:timestamp_end;0:~19:timestamp_tls_setup;0:~15:timestamp_start;0:~8:mitmcert;0:~15:tls_established;5:false!7:address;7:0:;1:0#]2:id;36:8b758ba3-70ce-49ac-873f-97a893864602;}5:error;0:~2:id;36:fac79186-100e-45ee-a9ab-f7369281cf50;}

test/mitmproxy/io/test_compat.py

@@ -9,6 +9,7 @@ from mitmproxy import exceptions
     ["dumpfile-018.mitm", "https://www.example.com/", 1],
     ["dumpfile-019.mitm", "https://webrv.rtb-seller.com/", 1],
     ["dumpfile-7-websocket.mitm", "https://echo.websocket.org/", 6],
+    ["dumpfile-10.mitm", "https://example.com/", 1]
 ])
 def test_load(tdata, dumpfile, url, count):
     with open(tdata.path("mitmproxy/data/" + dumpfile), "rb") as f:

test/mitmproxy/test_flow.py

@@ -34,7 +34,7 @@ class TestSerialize:
     def test_roundtrip(self):
         sio = io.BytesIO()
         f = tflow.tflow()
-        f.marked = True
+        f.marked = ":default:"
         f.request.content = bytes(range(256))
         w = mitmproxy.io.FlowWriter(sio)
         w.add(f)

test/mitmproxy/test_flowfilter.py

@@ -1,9 +1,7 @@
 import io
 import pytest
 from unittest.mock import patch
-
 from mitmproxy.test import tflow
-
 from mitmproxy import flowfilter, http
 
 
@@ -144,9 +142,16 @@ class TestMatchingHTTPFlow:
     def test_fmarked(self):
         q = self.req()
         assert not self.q("~marked", q)
-        q.marked = True
+        q.marked = ":default:"
         assert self.q("~marked", q)
 
+    def test_fmarker_char(self):
+        t = tflow.tflow()
+        t.marked = ":default:"
+        assert not self.q("~marker X", t)
+        t.marked = 'X'
+        assert self.q("~marker X", t)
+
     def test_head(self):
         q = self.req()
         s = self.resp()

test/mitmproxy/test_types.py

@@ -124,6 +124,25 @@ def test_cutspec():
         assert len(ret) == len(b.valid_prefixes)
 
 
+def test_marker():
+    with taddons.context() as tctx:
+        b = mitmproxy.types._MarkerType()
+        assert b.parse(tctx.master.commands, mitmproxy.types.Marker, ":red_circle:") == ":red_circle:"
+        assert b.parse(tctx.master.commands, mitmproxy.types.Marker, "true") == ":default:"
+        assert b.parse(tctx.master.commands, mitmproxy.types.Marker, "false") == ""
+
+        with pytest.raises(mitmproxy.exceptions.TypeError):
+            b.parse(tctx.master.commands, mitmproxy.types.Marker, ":bogus:")
+
+        assert b.is_valid(tctx.master.commands, mitmproxy.types.Marker, "true") is True
+        assert b.is_valid(tctx.master.commands, mitmproxy.types.Marker, "false") is True
+        assert b.is_valid(tctx.master.commands, mitmproxy.types.Marker, "bogus") is False
+        assert b.is_valid(tctx.master.commands, mitmproxy.types.Marker, "X") is True
+        assert b.is_valid(tctx.master.commands, mitmproxy.types.Marker, ":red_circle:") is True
+        ret = b.completion(tctx.master.commands, mitmproxy.types.Marker, ":smil")
+        assert len(ret) > 10
+
+
 def test_arg():
     with taddons.context() as tctx:
         b = mitmproxy.types._ArgType()

test/mitmproxy/utils/test_emoji.py

@@ -0,0 +1,6 @@
+from mitmproxy.utils import emoji
+from mitmproxy.tools.console.common import SYMBOL_MARK
+
+
+def test_emoji():
+    assert emoji.emoji[":default:"] == SYMBOL_MARK