From a70b50fe3a3065345cf26112a2148c648695b9d9 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 12 May 2018 11:05:01 +1200 Subject: [PATCH 1/6] console: implement positions for flow list walker This papers over an urwid bug until they fix it. fixes #2973 --- mitmproxy/tools/console/flowlist.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index e6bd1693c..a9e48af41 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -39,6 +39,14 @@ class FlowListWalker(urwid.ListWalker): def __init__(self, master): self.master = master + def positions(self, reverse=False): + # The stub implementation of positions can go once this issue is resolved: + # https://github.com/urwid/urwid/issues/294 + ret = range(len(self.master.view)) + if reverse: + return reversed(ret) + return ret + def view_changed(self): self._modified() From 9f83b6b944db0656372f8c5191670db493415452 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 12 May 2018 11:16:19 +1200 Subject: [PATCH 2/6] script: invoke addons as script with exception handler Fixes #2837 --- mitmproxy/addons/script.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index decd07596..706146b0a 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -156,7 +156,8 @@ class ScriptLoader: s = Script(path, False) for f in flows: for evt, arg in eventsequence.iterate(f): - ctx.master.addons.invoke_addon(s, evt, arg) + with addonmanager.safecall(): + ctx.master.addons.invoke_addon(s, evt, arg) except exceptions.OptionsError as e: script_error_handler(path, e, msg=str(e)) From 260f7290922a013274d701cec2601648f71c4d59 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 12 May 2018 11:39:44 +1200 Subject: [PATCH 3/6] Commit taken messages on kill Fixes 2879 --- mitmproxy/controller.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 582a6683f..6e2190663 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -113,6 +113,8 @@ class Reply: def kill(self, force=False): self.send(exceptions.Kill, force) + if self._state == "taken": + self.commit() def send(self, msg, force=False): if self.state not in {"start", "taken"}: From 88fe26997c9da8200722f900992fe1e6f021b34a Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 12 May 2018 12:09:48 +1200 Subject: [PATCH 4/6] script: revamp ephemeral script running Fixes 2838 --- mitmproxy/addons/script.py | 26 ++++++++++++++++++-------- test/mitmproxy/addons/test_script.py | 8 +++----- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 706146b0a..5c0fe2de8 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -148,16 +148,26 @@ class ScriptLoader: @command.command("script.run") def script_run(self, flows: typing.Sequence[flow.Flow], path: mtypes.Path) -> None: """ - Run a script on the specified flows. The script is loaded with - default options, and all lifecycle events for each flow are - simulated. + Run a script on the specified flows. The script is configured with + the current options and all lifecycle events for each flow are + simulated. Note that the load event is not invoked. """ + if not os.path.isfile(path): + ctx.log.error('No such script: %s' % path) + return try: - s = Script(path, False) - for f in flows: - for evt, arg in eventsequence.iterate(f): - with addonmanager.safecall(): - ctx.master.addons.invoke_addon(s, evt, arg) + mod = load_script(path) + if mod: + ctx.master.addons.invoke_addon(mod, "running") + ctx.master.addons.invoke_addon( + mod, + "configure", + ctx.options.keys() + ) + for f in flows: + for evt, arg in eventsequence.iterate(f): + with addonmanager.safecall(): + ctx.master.addons.invoke_addon(mod, evt, arg) except exceptions.OptionsError as e: script_error_handler(path, e, msg=str(e)) diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index 573b36e76..05472d9a0 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -173,16 +173,14 @@ class TestCutTraceback: class TestScriptLoader: @pytest.mark.asyncio async def test_script_run(self, tdata): - rp = tdata.path( - "mitmproxy/data/addonscripts/recorder/recorder.py" - ) + rp = tdata.path("mitmproxy/data/addonscripts/recorder/recorder.py") sc = script.ScriptLoader() with taddons.context(sc) as tctx: sc.script_run([tflow.tflow(resp=True)], rp) await tctx.master.await_log("recorder response") debug = [i.msg for i in tctx.master.logs if i.level == "debug"] assert debug == [ - 'recorder load', 'recorder running', 'recorder configure', + 'recorder running', 'recorder configure', 'recorder requestheaders', 'recorder request', 'recorder responseheaders', 'recorder response' ] @@ -192,7 +190,7 @@ class TestScriptLoader: sc = script.ScriptLoader() with taddons.context(sc) as tctx: sc.script_run([tflow.tflow(resp=True)], "/") - assert await tctx.master.await_log("/: No such script") + assert await tctx.master.await_log("No such script") def test_simple(self, tdata): sc = script.ScriptLoader() From 33eba29d1e6801af1b2e067503a2f505be3f489b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 12 May 2018 12:36:31 +1200 Subject: [PATCH 5/6] tcp: handle EINVAL from closed connections Fixes #2771 --- mitmproxy/net/tcp.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mitmproxy/net/tcp.py b/mitmproxy/net/tcp.py index 18429daa6..232840576 100644 --- a/mitmproxy/net/tcp.py +++ b/mitmproxy/net/tcp.py @@ -1,4 +1,5 @@ import os +import errno import select import socket import sys @@ -585,6 +586,13 @@ class TCPServer: with self.handler_counter: try: self.handle_client_connection(connection, client_address) + except OSError as e: + # This catches situations where the underlying connection is + # closed beneath us. Syscalls on the connection object at this + # point returns EINVAL. If this happens, we close the socket and + # move on. + if not e.errno == errno.EINVAL: + raise except: self.handle_error(connection, client_address) finally: From 94428b2ffdcb5780a7a12373472b6a21eb9fd83e Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 12 May 2018 13:06:08 +1200 Subject: [PATCH 6/6] script: tweak explicit command invocation --- mitmproxy/addons/script.py | 11 ++++------- mitmproxy/net/tcp.py | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 5c0fe2de8..a39ce5ce5 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -155,9 +155,9 @@ class ScriptLoader: if not os.path.isfile(path): ctx.log.error('No such script: %s' % path) return - try: - mod = load_script(path) - if mod: + mod = load_script(path) + if mod: + with addonmanager.safecall(): ctx.master.addons.invoke_addon(mod, "running") ctx.master.addons.invoke_addon( mod, @@ -166,10 +166,7 @@ class ScriptLoader: ) for f in flows: for evt, arg in eventsequence.iterate(f): - with addonmanager.safecall(): - ctx.master.addons.invoke_addon(mod, evt, arg) - except exceptions.OptionsError as e: - script_error_handler(path, e, msg=str(e)) + ctx.master.addons.invoke_addon(mod, evt, arg) def configure(self, updated): if "scripts" in updated: diff --git a/mitmproxy/net/tcp.py b/mitmproxy/net/tcp.py index 232840576..2496d47c7 100644 --- a/mitmproxy/net/tcp.py +++ b/mitmproxy/net/tcp.py @@ -586,7 +586,7 @@ class TCPServer: with self.handler_counter: try: self.handle_client_connection(connection, client_address) - except OSError as e: + except OSError as e: # pragma: no cover # This catches situations where the underlying connection is # closed beneath us. Syscalls on the connection object at this # point returns EINVAL. If this happens, we close the socket and