From 0b090f7ae1221eba3972c99b21cf3dc516420895 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 28 Apr 2017 17:01:48 +1200 Subject: [PATCH] Commands, core update event This patch: - Introduces a core update() event that should be invoked whenever flows are changed outside of the normal lifecycle. - Extend view.resolve to know about @all, which matches all flows in the view. - Add a core flow.resume comand, which resumes flows and broadcasts an update event. - Define flow list bindings for: A -> flow.resume @all a -> flow.resume @focus d -> view.remove @focus z -> view.remove @all --- mitmproxy/addons/core.py | 13 +++++ mitmproxy/addons/view.py | 86 +++++++++++++++------------- mitmproxy/command.py | 4 +- mitmproxy/eventsequence.py | 1 + mitmproxy/tools/console/flowlist.py | 34 ++++------- mitmproxy/tools/console/master.py | 4 ++ mitmproxy/tools/web/app.py | 18 +++--- test/mitmproxy/addons/test_view.py | 29 ++++++---- test/mitmproxy/test_command.py | 2 +- test/mitmproxy/tools/web/test_app.py | 2 +- 10 files changed, 104 insertions(+), 89 deletions(-) diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index 3f9cb15e1..5194c4ee1 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -1,6 +1,9 @@ +import typing + from mitmproxy import ctx from mitmproxy import exceptions from mitmproxy import command +from mitmproxy import flow class Core: @@ -16,3 +19,13 @@ class Core: ctx.options.set(spec) except exceptions.OptionsError as e: raise exceptions.CommandError(e) from e + + @command.command("flow.resume") + def resume(self, flows: typing.Sequence[flow.Flow]) -> None: + """ + Resume flows if they are intercepted. + """ + intercepted = [i for i in flows if i.intercepted] + for f in intercepted: + f.resume() + ctx.master.addons.trigger("update", intercepted) diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index f4082abe4..f2768a58f 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -18,6 +18,7 @@ import sortedcontainers import mitmproxy.flow from mitmproxy import flowfilter from mitmproxy import exceptions +from mitmproxy import command from mitmproxy import ctx from mitmproxy import http # noqa @@ -223,7 +224,7 @@ class View(collections.Sequence): self.filter = flt or matchall self._refilter() - def clear(self): + def clear(self) -> None: """ Clears both the store and view. """ @@ -256,42 +257,20 @@ class View(collections.Sequence): self.focus.flow = f self.sig_view_add.send(self, flow=f) - def remove(self, f: mitmproxy.flow.Flow): + @command.command("view.remove") + def remove(self, flows: typing.Sequence[mitmproxy.flow.Flow]) -> None: """ Removes the flow from the underlying store and the view. """ - if f.id in self._store: - if f in self._view: - self._view.remove(f) - self.sig_view_remove.send(self, flow=f) - del self._store[f.id] - self.sig_store_remove.send(self, flow=f) - - def update(self, f: mitmproxy.flow.Flow): - """ - Updates a flow. If the flow is not in the state, it's ignored. - """ - if f.id in self._store: - if self.filter(f): - if f not in self._view: - self._base_add(f) - if self.focus_follow: - self.focus.flow = f - self.sig_view_add.send(self, flow=f) - else: - # This is a tad complicated. The sortedcontainers - # implementation assumes that the order key is stable. If - # it changes mid-way Very Bad Things happen. We detect when - # this happens, and re-fresh the item. - self.order_key.refresh(f) - self.sig_view_update.send(self, flow=f) - else: - try: + for f in flows: + if f.id in self._store: + if f.killable: + f.kill() + if f in self._view: self._view.remove(f) self.sig_view_remove.send(self, flow=f) - except ValueError: - # The value was not in the view - pass + del self._store[f.id] + self.sig_store_remove.send(self, flow=f) def get_by_id(self, flow_id: str) -> typing.Optional[mitmproxy.flow.Flow]: """ @@ -322,10 +301,13 @@ class View(collections.Sequence): if "console_focus_follow" in updated: self.focus_follow = ctx.options.console_focus_follow + @command.command("view.resolve") def resolve(self, spec: str) -> typing.Sequence[mitmproxy.flow.Flow]: """ Resolve a flow list specification to an actual list of flows. """ + if spec == "@all": + return [i for i in self._store.values()] if spec == "@focus": return [self.focus.flow] if self.focus.flow else [] elif spec == "@shown": @@ -342,26 +324,50 @@ class View(collections.Sequence): raise exceptions.CommandError("Invalid flow filter: %s" % spec) return [i for i in self._store.values() if filt(i)] - def load(self, l): - l.add_command("console.resolve", self.resolve) - def request(self, f): self.add(f) def error(self, f): - self.update(f) + self.update([f]) def response(self, f): - self.update(f) + self.update([f]) def intercept(self, f): - self.update(f) + self.update([f]) def resume(self, f): - self.update(f) + self.update([f]) def kill(self, f): - self.update(f) + self.update([f]) + + def update(self, flows: typing.Sequence[mitmproxy.flow.Flow]) -> None: + """ + Updates a list of flows. If flow is not in the state, it's ignored. + """ + for f in flows: + if f.id in self._store: + if self.filter(f): + if f not in self._view: + self._base_add(f) + if self.focus_follow: + self.focus.flow = f + self.sig_view_add.send(self, flow=f) + else: + # This is a tad complicated. The sortedcontainers + # implementation assumes that the order key is stable. If + # it changes mid-way Very Bad Things happen. We detect when + # this happens, and re-fresh the item. + self.order_key.refresh(f) + self.sig_view_update.send(self, flow=f) + else: + try: + self._view.remove(f) + self.sig_view_remove.send(self, flow=f) + except ValueError: + # The value was not in the view + pass class Focus: diff --git a/mitmproxy/command.py b/mitmproxy/command.py index fa6e23ea6..7e877dd33 100644 --- a/mitmproxy/command.py +++ b/mitmproxy/command.py @@ -110,9 +110,9 @@ def parsearg(manager: CommandManager, spec: str, argtype: type) -> typing.Any: if argtype == str: return spec elif argtype == typing.Sequence[flow.Flow]: - return manager.call_args("console.resolve", [spec]) + return manager.call_args("view.resolve", [spec]) elif argtype == flow.Flow: - flows = manager.call_args("console.resolve", [spec]) + flows = manager.call_args("view.resolve", [spec]) if len(flows) != 1: raise exceptions.CommandError( "Command requires one flow, specification matched %s." % len(flows) diff --git a/mitmproxy/eventsequence.py b/mitmproxy/eventsequence.py index 30c037f14..4e1999726 100644 --- a/mitmproxy/eventsequence.py +++ b/mitmproxy/eventsequence.py @@ -35,6 +35,7 @@ Events = frozenset([ "load", "running", "tick", + "update", ]) diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index bb59a9b7f..f5ce02afe 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -150,14 +150,7 @@ class FlowItem(urwid.WidgetWrap): def keypress(self, xxx_todo_changeme, key): (maxcol,) = xxx_todo_changeme key = common.shortcuts(key) - if key == "a": - self.flow.resume() - self.master.view.update(self.flow) - elif key == "d": - if self.flow.killable: - self.flow.kill() - self.master.view.remove(self.flow) - elif key == "D": + if key == "D": cp = self.flow.copy() self.master.view.add(cp) self.master.view.focus.flow = cp @@ -222,14 +215,14 @@ class FlowItem(urwid.WidgetWrap): callback = common.export_to_clip_or_file, args = (None, self.flow, common.ask_save_path) ) - elif key == "C": - signals.status_prompt_onekey.send( - self, - prompt = "Export to clipboard", - keys = [(e[0], e[1]) for e in export.EXPORTERS], - callback = common.export_to_clip_or_file, - args = (None, self.flow, common.copy_to_clipboard_or_prompt) - ) + # elif key == "C": + # signals.status_prompt_onekey.send( + # self, + # prompt = "Export to clipboard", + # keys = [(e[0], e[1]) for e in export.EXPORTERS], + # callback = common.export_to_clip_or_file, + # args = (None, self.flow, common.copy_to_clipboard_or_prompt) + # ) elif key == "b": common.ask_save_body(None, self.flow) else: @@ -321,14 +314,7 @@ class FlowListBox(urwid.ListBox): def keypress(self, size, key): key = common.shortcuts(key) - if key == "A": - for f in self.master.view: - if f.intercepted: - f.resume() - self.master.view.update(f) - elif key == "z": - self.master.view.clear() - elif key == "Z": + if key == "Z": self.master.view.clear_not_marked() elif key == "g": if len(self.master.view): diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 7787ba11f..59203d04a 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -147,11 +147,15 @@ def default_keymap(km): km.add("i", "console.command 'set intercept='") km.add("W", "console.command 'set save_stream_file='") + km.add("A", "flow.resume @all", context="flowlist") + km.add("a", "flow.resume @focus", context="flowlist") + km.add("d", "view.remove @focus", context="flowlist") km.add("F", "set console_focus_follow=toggle", context="flowlist") km.add("v", "set console_order_reversed=toggle", context="flowlist") km.add("f", "console.command 'set view_filter='", context="flowlist") km.add("e", "set console_eventlog=toggle", context="flowlist") km.add("w", "console.command 'save.file @shown '", context="flowlist") + km.add("z", "view.remove @all", context="flowlist") km.add("enter", "console.view.flow @focus", context="flowlist") diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 1e01c57cb..7302cea42 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -246,7 +246,7 @@ class ResumeFlows(RequestHandler): def post(self): for f in self.view: f.resume() - self.view.update(f) + self.view.update([f]) class KillFlows(RequestHandler): @@ -254,27 +254,27 @@ class KillFlows(RequestHandler): for f in self.view: if f.killable: f.kill() - self.view.update(f) + self.view.update([f]) class ResumeFlow(RequestHandler): def post(self, flow_id): self.flow.resume() - self.view.update(self.flow) + self.view.update([self.flow]) class KillFlow(RequestHandler): def post(self, flow_id): if self.flow.killable: self.flow.kill() - self.view.update(self.flow) + self.view.update([self.flow]) class FlowHandler(RequestHandler): def delete(self, flow_id): if self.flow.killable: self.flow.kill() - self.view.remove(self.flow) + self.view.remove([self.flow]) def put(self, flow_id): flow = self.flow @@ -317,7 +317,7 @@ class FlowHandler(RequestHandler): except APIError: flow.revert() raise - self.view.update(flow) + self.view.update([flow]) class DuplicateFlow(RequestHandler): @@ -331,14 +331,14 @@ class RevertFlow(RequestHandler): def post(self, flow_id): if self.flow.modified(): self.flow.revert() - self.view.update(self.flow) + self.view.update([self.flow]) class ReplayFlow(RequestHandler): def post(self, flow_id): self.flow.backup() self.flow.response = None - self.view.update(self.flow) + self.view.update([self.flow]) try: self.master.replay_request(self.flow) @@ -351,7 +351,7 @@ class FlowContent(RequestHandler): self.flow.backup() message = getattr(self.flow, message) message.content = self.filecontents - self.view.update(self.flow) + self.view.update([self.flow]) def get(self, flow_id, message): message = getattr(self.flow, message) diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 05d4af30d..878faac11 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -31,7 +31,7 @@ def test_order_refresh(): v.add(tf) tf.request.timestamp_start = 1 assert not sargs - v.update(tf) + v.update([tf]) assert sargs @@ -140,6 +140,7 @@ def test_load(): def test_resolve(): v = view.View() with taddons.context(options=options.Options()) as tctx: + assert tctx.command(v.resolve, "@all") == [] assert tctx.command(v.resolve, "@focus") == [] assert tctx.command(v.resolve, "@shown") == [] assert tctx.command(v.resolve, "@hidden") == [] @@ -149,6 +150,7 @@ def test_resolve(): v.request(tft(method="get")) assert len(tctx.command(v.resolve, "~m get")) == 1 assert len(tctx.command(v.resolve, "@focus")) == 1 + assert len(tctx.command(v.resolve, "@all")) == 1 assert len(tctx.command(v.resolve, "@shown")) == 1 assert len(tctx.command(v.resolve, "@unmarked")) == 1 assert tctx.command(v.resolve, "@hidden") == [] @@ -156,6 +158,7 @@ def test_resolve(): v.request(tft(method="put")) assert len(tctx.command(v.resolve, "@focus")) == 1 assert len(tctx.command(v.resolve, "@shown")) == 2 + assert len(tctx.command(v.resolve, "@all")) == 2 assert tctx.command(v.resolve, "@hidden") == [] assert tctx.command(v.resolve, "@marked") == [] @@ -175,6 +178,7 @@ def test_resolve(): assert m(tctx.command(v.resolve, "@hidden")) == ["PUT", "PUT"] assert m(tctx.command(v.resolve, "@marked")) == ["GET"] assert m(tctx.command(v.resolve, "@unmarked")) == ["PUT", "GET", "PUT"] + assert m(tctx.command(v.resolve, "@all")) == ["GET", "PUT", "GET", "PUT"] with pytest.raises(exceptions.CommandError, match="Invalid flow filter"): tctx.command(v.resolve, "~") @@ -230,14 +234,14 @@ def test_update(): assert f in v f.request.method = "put" - v.update(f) + v.update([f]) assert f not in v f.request.method = "get" - v.update(f) + v.update([f]) assert f in v - v.update(f) + v.update([f]) assert f in v @@ -291,14 +295,14 @@ def test_signals(): # An update that results in a flow being added to the view clearrec() v[0].request.method = "PUT" - v.update(v[0]) + v.update([v[0]]) assert rec_remove assert not any([rec_update, rec_refresh, rec_add]) # An update that does not affect the view just sends update v.set_filter(flowfilter.parse("~m put")) clearrec() - v.update(v[0]) + v.update([v[0]]) assert rec_update assert not any([rec_remove, rec_refresh, rec_add]) @@ -307,7 +311,7 @@ def test_signals(): v.set_filter(flowfilter.parse("~m get")) assert not len(v) clearrec() - v.update(f) + v.update([f]) assert not any([rec_add, rec_update, rec_remove, rec_refresh]) @@ -333,7 +337,7 @@ def test_focus_follow(): assert v.focus.flow.request.timestamp_start == 7 mod.request.method = "GET" - v.update(mod) + v.update([mod]) assert v.focus.index == 2 assert v.focus.flow.request.timestamp_start == 6 @@ -374,15 +378,16 @@ def test_focus(): assert f.index == 0 f.index = 1 - v.remove(v[1]) + v.remove([v[1]]) + v[1].intercept() assert f.index == 1 assert f.flow is v[1] - v.remove(v[1]) + v.remove([v[1]]) assert f.index == 0 assert f.flow is v[0] - v.remove(v[0]) + v.remove([v[0]]) assert f.index is None assert f.flow is None @@ -413,7 +418,7 @@ def test_settings(): v.settings[f]["foo"] = "bar" assert v.settings[f]["foo"] == "bar" assert len(list(v.settings)) == 1 - v.remove(f) + v.remove([f]) with pytest.raises(KeyError): v.settings[f] assert not v.settings.keys() diff --git a/test/mitmproxy/test_command.py b/test/mitmproxy/test_command.py index 64928dbf6..47feaacf5 100644 --- a/test/mitmproxy/test_command.py +++ b/test/mitmproxy/test_command.py @@ -65,7 +65,7 @@ def test_typename(): class DummyConsole: def load(self, l): - l.add_command("console.resolve", self.resolve) + l.add_command("view.resolve", self.resolve) def resolve(self, spec: str) -> typing.Sequence[flow.Flow]: n = int(spec) diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py index e3d5dc44e..61c262940 100644 --- a/test/mitmproxy/tools/web/test_app.py +++ b/test/mitmproxy/tools/web/test_app.py @@ -162,7 +162,7 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): f = self.view.get_by_id(resp.body.decode()) assert f assert f.id != "42" - self.view.remove(f) + self.view.remove([f]) def test_flow_revert(self): f = self.view.get_by_id("42")