From 4522a6f7b754be26084c40df5ecc7349023a692e Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 12:13:52 +1300 Subject: [PATCH 01/12] Start moving addon options into /addons This takes the first few steps: - Extends taddons to make loading addon options easier - Removes dependencies in the test suite on options in addons - Tweaks command-line parser autocreation to ignore nonexistent options. This lets us load common options without over-depending on loaded addons. --- mitmproxy/addons/anticache.py | 9 +++++++++ mitmproxy/addons/anticomp.py | 6 ++++++ mitmproxy/options.py | 13 ------------- mitmproxy/optmanager.py | 7 +++++++ mitmproxy/test/taddons.py | 6 +++++- test/mitmproxy/addons/test_anticache.py | 2 +- test/mitmproxy/addons/test_anticomp.py | 2 +- test/mitmproxy/addons/test_core.py | 6 +++--- test/mitmproxy/test_optmanager.py | 7 ++++--- test/mitmproxy/tools/console/test_statusbar.py | 8 ++++---- 10 files changed, 40 insertions(+), 26 deletions(-) diff --git a/mitmproxy/addons/anticache.py b/mitmproxy/addons/anticache.py index 5b34d5a51..9f5c2dc1d 100644 --- a/mitmproxy/addons/anticache.py +++ b/mitmproxy/addons/anticache.py @@ -2,6 +2,15 @@ from mitmproxy import ctx class AntiCache: + def load(self, loader): + loader.add_option( + "anticache", bool, False, + """ + Strip out request headers that might cause the server to return + 304-not-modified. + """ + ) + def request(self, flow): if ctx.options.anticache: flow.request.anticache() diff --git a/mitmproxy/addons/anticomp.py b/mitmproxy/addons/anticomp.py index d7d1ca8da..3415302a6 100644 --- a/mitmproxy/addons/anticomp.py +++ b/mitmproxy/addons/anticomp.py @@ -2,6 +2,12 @@ from mitmproxy import ctx class AntiComp: + def load(self, loader): + loader.add_option( + "anticomp", bool, False, + "Try to convince servers to send us un-compressed data." + ) + def request(self, flow): if ctx.options.anticomp: flow.request.anticomp() diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 76060548a..f31b4a036 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -31,8 +31,6 @@ class Options(optmanager.OptManager): # Autogenerated using test/helper_tools/typehints_for_options.py add_upstream_certs_to_client_chain = None # type: bool allow_remote = None # type: bool - anticache = None # type: bool - anticomp = None # type: bool body_size_limit = None # type: Optional[str] cadir = None # type: str certs = None # type: Sequence[str] @@ -121,17 +119,6 @@ class Options(optmanager.OptManager): "onboarding_port", int, APP_PORT, "Port to serve the onboarding app from." ) - self.add_option( - "anticache", bool, False, - """ - Strip out request headers that might cause the server to return - 304-not-modified. - """ - ) - self.add_option( - "anticomp", bool, False, - "Try to convince servers to send us un-compressed data." - ) self.add_option( "client_replay", Sequence[str], [], "Replay client requests from a saved file." diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 01d97af31..bb9e30302 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -327,6 +327,13 @@ class OptManager: return d def make_parser(self, parser, optname, metavar=None, short=None): + """ + Auto-Create a command-line parser entry for a named option. If the + option does not exist, it is ignored. + """ + if optname not in self._options: + return + o = self._options[optname] def mkf(l, s): diff --git a/mitmproxy/test/taddons.py b/mitmproxy/test/taddons.py index d966f1d5d..12fc09861 100644 --- a/mitmproxy/test/taddons.py +++ b/mitmproxy/test/taddons.py @@ -59,7 +59,7 @@ class context: provides a number of helper methods for common testing scenarios. """ - def __init__(self, master=None, options=None): + def __init__(self, *addons, master=None, options=None): options = options or mitmproxy.options.Options() self.master = master or RecordingMaster( options @@ -67,6 +67,10 @@ class context: self.options = self.master.options self.wrapped = None + loader = addonmanager.Loader(self.master) + for a in addons: + self.master.addons.invoke_addon(a, "load", loader) + def ctx(self): """ Returns a new handler context. diff --git a/test/mitmproxy/addons/test_anticache.py b/test/mitmproxy/addons/test_anticache.py index 928f2180f..d1765fe07 100644 --- a/test/mitmproxy/addons/test_anticache.py +++ b/test/mitmproxy/addons/test_anticache.py @@ -7,7 +7,7 @@ from mitmproxy.test import taddons class TestAntiCache: def test_simple(self): sa = anticache.AntiCache() - with taddons.context() as tctx: + with taddons.context(sa) as tctx: f = tflow.tflow(resp=True) f.request.headers["if-modified-since"] = "test" f.request.headers["if-none-match"] = "test" diff --git a/test/mitmproxy/addons/test_anticomp.py b/test/mitmproxy/addons/test_anticomp.py index 2a6cf3ce3..92650332c 100644 --- a/test/mitmproxy/addons/test_anticomp.py +++ b/test/mitmproxy/addons/test_anticomp.py @@ -7,7 +7,7 @@ from mitmproxy.test import taddons class TestAntiComp: def test_simple(self): sa = anticomp.AntiComp() - with taddons.context() as tctx: + with taddons.context(sa) as tctx: f = tflow.tflow(resp=True) sa.request(f) diff --git a/test/mitmproxy/addons/test_core.py b/test/mitmproxy/addons/test_core.py index 5aa4ef376..5c9a8a0d2 100644 --- a/test/mitmproxy/addons/test_core.py +++ b/test/mitmproxy/addons/test_core.py @@ -10,9 +10,9 @@ def test_set(): with taddons.context() as tctx: tctx.master.addons.add(sa) - assert not tctx.master.options.anticomp - tctx.command(sa.set, "anticomp") - assert tctx.master.options.anticomp + assert tctx.master.options.server + tctx.command(sa.set, "server=false") + assert not tctx.master.options.server with pytest.raises(exceptions.CommandError): tctx.command(sa.set, "nonexistent") diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index d9b932277..cd8857cab 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -351,7 +351,7 @@ def test_dump_defaults(): def test_dump_dicts(): o = options.Options() assert optmanager.dump_dicts(o) - assert optmanager.dump_dicts(o, ['http2', 'anticomp']) + assert optmanager.dump_dicts(o, ['http2', 'listen_port']) class TTypes(optmanager.OptManager): @@ -375,8 +375,9 @@ def test_make_parser(): opts.make_parser(parser, "int", short="c") opts.make_parser(parser, "seqstr", short="d") opts.make_parser(parser, "bool_on", short="e") - with pytest.raises(ValueError): - opts.make_parser(parser, "unknown") + + # No error for nonexistent options + opts.make_parser(parser, "xxxxxxx") def test_set(): diff --git a/test/mitmproxy/tools/console/test_statusbar.py b/test/mitmproxy/tools/console/test_statusbar.py index 8522eb96a..ac17c5c08 100644 --- a/test/mitmproxy/tools/console/test_statusbar.py +++ b/test/mitmproxy/tools/console/test_statusbar.py @@ -3,7 +3,9 @@ from mitmproxy.tools.console import statusbar, master def test_statusbar(monkeypatch): - o = options.Options( + o = options.Options() + m = master.ConsoleMaster(o) + m.options.update( setheaders=[":~q:foo:bar"], replacements=[":~q:foo:bar"], ignore_hosts=["example.com", "example.org"], @@ -21,10 +23,8 @@ def test_statusbar(monkeypatch): upstream_cert=False, stream_large_bodies="3m", mode="transparent", - scripts=["nonexistent"], - save_stream_file="foo", ) - m = master.ConsoleMaster(o) + m.options.update(view_order='url', console_focus_follow=True) monkeypatch.setattr(m.addons.get("clientplayback"), "count", lambda: 42) monkeypatch.setattr(m.addons.get("serverplayback"), "count", lambda: 42) From af39e022ef6c03b194fbee4a7091aed2bf72a652 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 12:16:31 +1300 Subject: [PATCH 02/12] Add .pytest_cache to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9fade1c30..1bb90c1ac 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ MANIFEST build/ dist/ mitmproxy/contrib/kaitaistruct/*.ksy +.pytest_cache # UI From 42094b29ff5e042b3cf3c2544fc7a60555cd3c26 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 12:33:48 +1300 Subject: [PATCH 03/12] Clean up un-necessary arguments to taddons.context Also test coverage ++ --- test/helper_tools/dumperview.py | 3 +-- test/mitmproxy/addons/test_dumper.py | 15 +++++++-------- test/mitmproxy/addons/test_intercept.py | 3 +-- test/mitmproxy/addons/test_save.py | 3 +-- test/mitmproxy/test_optmanager.py | 7 +++++-- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/test/helper_tools/dumperview.py b/test/helper_tools/dumperview.py index d417d767b..e17dc77bb 100755 --- a/test/helper_tools/dumperview.py +++ b/test/helper_tools/dumperview.py @@ -4,12 +4,11 @@ import click from mitmproxy.addons import dumper from mitmproxy.test import tflow from mitmproxy.test import taddons -from mitmproxy.tools import options def show(flow_detail, flows): d = dumper.Dumper() - with taddons.context(options=options.Options()) as ctx: + with taddons.context() as ctx: ctx.configure(d, flow_detail=flow_detail) for f in flows: ctx.cycle(d, f) diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index fb80f3cee..9774e1316 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -10,12 +10,11 @@ from mitmproxy.test import tutils from mitmproxy.addons import dumper from mitmproxy import exceptions from mitmproxy import http -from mitmproxy import options def test_configure(): d = dumper.Dumper() - with taddons.context(options=options.Options()) as ctx: + with taddons.context() as ctx: ctx.configure(d, view_filter="~b foo") assert d.filter @@ -34,7 +33,7 @@ def test_configure(): def test_simple(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: + with taddons.context() as ctx: ctx.configure(d, flow_detail=0) d.response(tflow.tflow(resp=True)) assert not sio.getvalue() @@ -102,7 +101,7 @@ def test_echo_body(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: + with taddons.context() as ctx: ctx.configure(d, flow_detail=3) d._echo_message(f.response) t = sio.getvalue() @@ -112,7 +111,7 @@ def test_echo_body(): def test_echo_request_line(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: + with taddons.context() as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.tflow(client_conn=None, server_conn=True, resp=True) f.request.is_replay = True @@ -147,7 +146,7 @@ class TestContentView: view_auto.side_effect = exceptions.ContentViewException("") sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: + with taddons.context() as ctx: ctx.configure(d, flow_detail=4, verbosity='debug') d.response(tflow.tflow()) assert ctx.master.has_log("content viewer failed") @@ -156,7 +155,7 @@ class TestContentView: def test_tcp(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: + with taddons.context() as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.ttcpflow() d.tcp_message(f) @@ -171,7 +170,7 @@ def test_tcp(): def test_websocket(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: + with taddons.context() as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.twebsocketflow() d.websocket_message(f) diff --git a/test/mitmproxy/addons/test_intercept.py b/test/mitmproxy/addons/test_intercept.py index d4999eb5f..d95981010 100644 --- a/test/mitmproxy/addons/test_intercept.py +++ b/test/mitmproxy/addons/test_intercept.py @@ -1,7 +1,6 @@ import pytest from mitmproxy.addons import intercept -from mitmproxy import options from mitmproxy import exceptions from mitmproxy.test import taddons from mitmproxy.test import tflow @@ -9,7 +8,7 @@ from mitmproxy.test import tflow def test_simple(): r = intercept.Intercept() - with taddons.context(options=options.Options()) as tctx: + with taddons.context() as tctx: assert not r.filt tctx.configure(r, intercept="~q") assert r.filt diff --git a/test/mitmproxy/addons/test_save.py b/test/mitmproxy/addons/test_save.py index 2dee708fb..4486ff783 100644 --- a/test/mitmproxy/addons/test_save.py +++ b/test/mitmproxy/addons/test_save.py @@ -5,14 +5,13 @@ from mitmproxy.test import tflow from mitmproxy import io from mitmproxy import exceptions -from mitmproxy import options from mitmproxy.addons import save from mitmproxy.addons import view def test_configure(tmpdir): sa = save.Save() - with taddons.context(options=options.Options()) as tctx: + with taddons.context() as tctx: with pytest.raises(exceptions.OptionsError): tctx.configure(sa, save_stream_file=str(tmpdir)) with pytest.raises(Exception, match="Invalid filter"): diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index cd8857cab..1c49c0b80 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -376,8 +376,11 @@ def test_make_parser(): opts.make_parser(parser, "seqstr", short="d") opts.make_parser(parser, "bool_on", short="e") - # No error for nonexistent options - opts.make_parser(parser, "xxxxxxx") + with pytest.raises(ValueError): + opts.make_parser(parser, "unknown") + + # Nonexistent options ignore + opts.make_parser(parser, "nonexistentxxx") def test_set(): From a12d1492308f281ae599f1142104eb62d924f3f1 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 14:39:03 +1300 Subject: [PATCH 04/12] client options: migrate allowremote and clientplayback Also triage options, and categorize them into core options (won't be migrated), options that are hard to migrate for various reasons, and easy migrations. --- mitmproxy/addons/clientplayback.py | 6 ++ mitmproxy/options.py | 80 +++++++++----------- test/mitmproxy/addons/test_allowremote.py | 3 +- test/mitmproxy/addons/test_clientplayback.py | 6 +- 4 files changed, 47 insertions(+), 48 deletions(-) diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 2dd488b9e..a017ec0f7 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -14,6 +14,12 @@ class ClientPlayback: self.current_thread = None self.configured = False + def load(self, loader): + loader.add_option( + "client_replay", typing.Sequence[str], [], + "Replay client requests from a saved file." + ) + def count(self) -> int: if self.current_thread: current = 1 diff --git a/mitmproxy/options.py b/mitmproxy/options.py index f31b4a036..ff13c385b 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -30,77 +30,75 @@ class Options(optmanager.OptManager): # This provides type hints for IDEs (e.g. PyCharm) and mypy. # Autogenerated using test/helper_tools/typehints_for_options.py add_upstream_certs_to_client_chain = None # type: bool - allow_remote = None # type: bool body_size_limit = None # type: Optional[str] cadir = None # type: str certs = None # type: Sequence[str] ciphers_client = None # type: Optional[str] ciphers_server = None # type: Optional[str] client_certs = None # type: Optional[str] - client_replay = None # type: Sequence[str] - console_focus_follow = None # type: bool - console_layout = None # type: str - console_layout_headers = None # type: bool - console_mouse = None # type: bool - console_palette = None # type: str - console_palette_transparent = None # type: bool - default_contentview = None # type: str - flow_detail = None # type: int http2 = None # type: bool http2_priority = None # type: bool ignore_hosts = None # type: Sequence[str] - intercept = None # type: Optional[str] - intercept_active = None # type: bool keep_host_header = None # type: bool - keepserving = None # type: bool listen_host = None # type: str listen_port = None # type: int mode = None # type: str - onboarding = None # type: bool - onboarding_host = None # type: str - onboarding_port = None # type: int - proxyauth = None # type: Optional[str] rawtcp = None # type: bool - server_replay_refresh = None # type: bool - replacements = None # type: Sequence[str] - server_replay_kill_extra = None # type: bool - rfile = None # type: Optional[str] - save_stream_file = None # type: Optional[str] - save_stream_filter = None # type: Optional[str] - scripts = None # type: Sequence[str] server = None # type: bool - server_replay = None # type: Sequence[str] - server_replay_ignore_content = None # type: bool - server_replay_ignore_host = None # type: bool - server_replay_ignore_params = None # type: Sequence[str] - server_replay_ignore_payload_params = None # type: Sequence[str] - server_replay_nopop = None # type: bool - server_replay_use_headers = None # type: Sequence[str] - setheaders = None # type: Sequence[str] - showhost = None # type: bool spoof_source_address = None # type: bool ssl_insecure = None # type: bool ssl_verify_upstream_trusted_ca = None # type: Optional[str] ssl_verify_upstream_trusted_cadir = None # type: Optional[str] ssl_version_client = None # type: str ssl_version_server = None # type: str + tcp_hosts = None # type: Sequence[str] + upstream_bind_address = None # type: str + upstream_cert = None # type: bool + websocket = None # type: bool + + # FIXME: Options that must be migrated to addons, but are complicated + # because they're used by more than one addon, or because they're + # embedded in the core code somehow. + default_contentview = None # type: str + flow_detail = None # type: int + intercept = None # type: Optional[str] + intercept_active = None # type: bool + proxyauth = None # type: Optional[str] + showhost = None # type: bool + verbosity = None # type: str + view_filter = None # type: Optional[str] + + # FIXME: Options that should be uncomplicated to migrate to addons + keepserving = None # type: bool + onboarding = None # type: bool + onboarding_host = None # type: str + onboarding_port = None # type: int + server_replay_refresh = None # type: bool + replacements = None # type: Sequence[str] + rfile = None # type: Optional[str] + save_stream_file = None # type: Optional[str] + save_stream_filter = None # type: Optional[str] + scripts = None # type: Sequence[str] + server_replay = None # type: Sequence[str] + server_replay_ignore_content = None # type: bool + server_replay_ignore_host = None # type: bool + server_replay_ignore_params = None # type: Sequence[str] + server_replay_ignore_payload_params = None # type: Sequence[str] + server_replay_kill_extra = None # type: bool + server_replay_nopop = None # type: bool + server_replay_use_headers = None # type: Sequence[str] + setheaders = None # type: Sequence[str] stickyauth = None # type: Optional[str] stickycookie = None # type: Optional[str] stream_large_bodies = None # type: Optional[str] stream_websockets = None # type: bool - tcp_hosts = None # type: Sequence[str] upstream_auth = None # type: Optional[str] - upstream_bind_address = None # type: str - upstream_cert = None # type: bool - verbosity = None # type: str - view_filter = None # type: Optional[str] view_order = None # type: str view_order_reversed = None # type: bool web_debug = None # type: bool web_iface = None # type: str web_open_browser = None # type: bool web_port = None # type: int - websocket = None # type: bool def __init__(self, **kwargs) -> None: super().__init__() @@ -119,10 +117,6 @@ class Options(optmanager.OptManager): "onboarding_port", int, APP_PORT, "Port to serve the onboarding app from." ) - self.add_option( - "client_replay", Sequence[str], [], - "Replay client requests from a saved file." - ) self.add_option( "server_replay_kill_extra", bool, False, "Kill extra requests during replay." diff --git a/test/mitmproxy/addons/test_allowremote.py b/test/mitmproxy/addons/test_allowremote.py index 9fc715251..52dae68d4 100644 --- a/test/mitmproxy/addons/test_allowremote.py +++ b/test/mitmproxy/addons/test_allowremote.py @@ -19,8 +19,7 @@ from mitmproxy.test import taddons ]) def test_allowremote(allow_remote, ip, should_be_killed): ar = allowremote.AllowRemote() - with taddons.context() as tctx: - tctx.master.addons.register(ar) + with taddons.context(ar) as tctx: tctx.options.allow_remote = allow_remote with mock.patch('mitmproxy.proxy.protocol.base.Layer') as layer: diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py index 3f990668b..f172af83e 100644 --- a/test/mitmproxy/addons/test_clientplayback.py +++ b/test/mitmproxy/addons/test_clientplayback.py @@ -24,7 +24,7 @@ class MockThread(): class TestClientPlayback: def test_playback(self): cp = clientplayback.ClientPlayback() - with taddons.context() as tctx: + with taddons.context(cp) as tctx: assert cp.count() == 0 f = tflow.tflow(resp=True) cp.start_replay([f]) @@ -58,7 +58,7 @@ class TestClientPlayback: def test_load_file(self, tmpdir): cp = clientplayback.ClientPlayback() - with taddons.context(): + with taddons.context(cp): fpath = str(tmpdir.join("flows")) tdump(fpath, [tflow.tflow(resp=True)]) cp.load_file(fpath) @@ -68,7 +68,7 @@ class TestClientPlayback: def test_configure(self, tmpdir): cp = clientplayback.ClientPlayback() - with taddons.context() as tctx: + with taddons.context(cp) as tctx: path = str(tmpdir.join("flows")) tdump(path, [tflow.tflow()]) tctx.configure(cp, client_replay=[path]) From 74a0230788f1e3a1800d8125fca274cda402c769 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 14:53:54 +1300 Subject: [PATCH 05/12] addon options: shift keepserving and onboarding options --- mitmproxy/addons/keepserving.py | 10 ++++++++ mitmproxy/addons/onboarding.py | 20 ++++++++++++++++ mitmproxy/options.py | 29 ----------------------- test/mitmproxy/addons/test_keepserving.py | 3 +-- test/mitmproxy/addons/test_onboarding.py | 18 +++++++------- test/mitmproxy/tservers.py | 4 ++-- 6 files changed, 43 insertions(+), 41 deletions(-) diff --git a/mitmproxy/addons/keepserving.py b/mitmproxy/addons/keepserving.py index 9c975a7b0..6413299d8 100644 --- a/mitmproxy/addons/keepserving.py +++ b/mitmproxy/addons/keepserving.py @@ -2,6 +2,16 @@ from mitmproxy import ctx class KeepServing: + def load(self, loader): + loader.add_option( + "keepserving", bool, False, + """ + Continue serving after client playback, server playback or file + read. This option is ignored by interactive tools, which always keep + serving. + """ + ) + def event_processing_complete(self): if not ctx.master.options.keepserving: ctx.master.shutdown() diff --git a/mitmproxy/addons/onboarding.py b/mitmproxy/addons/onboarding.py index 07536c347..900acb086 100644 --- a/mitmproxy/addons/onboarding.py +++ b/mitmproxy/addons/onboarding.py @@ -2,6 +2,9 @@ from mitmproxy.addons import wsgiapp from mitmproxy.addons.onboardingapp import app from mitmproxy import ctx +APP_HOST = "mitm.it" +APP_PORT = 80 + class Onboarding(wsgiapp.WSGIApp): name = "onboarding" @@ -9,6 +12,23 @@ class Onboarding(wsgiapp.WSGIApp): def __init__(self): super().__init__(app.Adapter(app.application), None, None) + def load(self, loader): + loader.add_option( + "onboarding", bool, True, + "Toggle the mitmproxy onboarding app." + ) + loader.add_option( + "onboarding_host", str, APP_HOST, + """ + Onboarding app domain. For transparent mode, use an IP when a DNS + entry for the app domain is not present. + """ + ) + loader.add_option( + "onboarding_port", int, APP_PORT, + "Port to serve the onboarding app from." + ) + def configure(self, updated): self.host = ctx.options.onboarding_host self.port = ctx.options.onboarding_port diff --git a/mitmproxy/options.py b/mitmproxy/options.py index ff13c385b..318458581 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -12,8 +12,6 @@ log_verbosity = [ "debug", ] -APP_HOST = "mitm.it" -APP_PORT = 80 CA_DIR = "~/.mitmproxy" LISTEN_PORT = 8080 @@ -69,10 +67,6 @@ class Options(optmanager.OptManager): view_filter = None # type: Optional[str] # FIXME: Options that should be uncomplicated to migrate to addons - keepserving = None # type: bool - onboarding = None # type: bool - onboarding_host = None # type: str - onboarding_port = None # type: int server_replay_refresh = None # type: bool replacements = None # type: Sequence[str] rfile = None # type: Optional[str] @@ -102,33 +96,10 @@ class Options(optmanager.OptManager): def __init__(self, **kwargs) -> None: super().__init__() - self.add_option( - "onboarding", bool, True, - "Toggle the mitmproxy onboarding app." - ) - self.add_option( - "onboarding_host", str, APP_HOST, - """ - Onboarding app domain. For transparent mode, use an IP when a DNS - entry for the app domain is not present. - """ - ) - self.add_option( - "onboarding_port", int, APP_PORT, - "Port to serve the onboarding app from." - ) self.add_option( "server_replay_kill_extra", bool, False, "Kill extra requests during replay." ) - self.add_option( - "keepserving", bool, False, - """ - Continue serving after client playback, server playback or file - read. This option is ignored by interactive tools, which always keep - serving. - """ - ) self.add_option( "server", bool, True, "Start a proxy server. Enabled by default." diff --git a/test/mitmproxy/addons/test_keepserving.py b/test/mitmproxy/addons/test_keepserving.py index 70f7e1e61..2869d0976 100644 --- a/test/mitmproxy/addons/test_keepserving.py +++ b/test/mitmproxy/addons/test_keepserving.py @@ -4,7 +4,6 @@ from mitmproxy.test import taddons def test_keepserving(): ks = keepserving.KeepServing() - - with taddons.context() as tctx: + with taddons.context(ks) as tctx: ks.event_processing_complete() assert tctx.master.should_exit.is_set() diff --git a/test/mitmproxy/addons/test_onboarding.py b/test/mitmproxy/addons/test_onboarding.py index 474e6c3cf..810ddef1d 100644 --- a/test/mitmproxy/addons/test_onboarding.py +++ b/test/mitmproxy/addons/test_onboarding.py @@ -2,7 +2,6 @@ import pytest from mitmproxy.addons import onboarding from mitmproxy.test import taddons -from mitmproxy import options from .. import tservers @@ -11,25 +10,28 @@ class TestApp(tservers.HTTPProxyTest): return [onboarding.Onboarding()] def test_basic(self): - with taddons.context() as tctx: - tctx.configure(self.addons()[0]) + ob = onboarding.Onboarding() + with taddons.context(ob) as tctx: + tctx.configure(ob) assert self.app("/").status_code == 200 @pytest.mark.parametrize("ext", ["pem", "p12"]) def test_cert(self, ext): - with taddons.context() as tctx: - tctx.configure(self.addons()[0]) + ob = onboarding.Onboarding() + with taddons.context(ob) as tctx: + tctx.configure(ob) resp = self.app("/cert/%s" % ext) assert resp.status_code == 200 assert resp.content @pytest.mark.parametrize("ext", ["pem", "p12"]) def test_head(self, ext): - with taddons.context() as tctx: - tctx.configure(self.addons()[0]) + ob = onboarding.Onboarding() + with taddons.context(ob) as tctx: + tctx.configure(ob) p = self.pathoc() with p.connect(): - resp = p.request("head:'http://%s/cert/%s'" % (options.APP_HOST, ext)) + resp = p.request("head:'http://%s/cert/%s'" % (tctx.options.onboarding_host, ext)) assert resp.status_code == 200 assert "Content-Length" in resp.headers assert not resp.content diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index dd5bb3276..4363931f7 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -222,12 +222,12 @@ class HTTPProxyTest(ProxyTestBase): p = pathod.pathoc.Pathoc( ("127.0.0.1", self.proxy.port), True, fp=None ) - with p.connect((options.APP_HOST, options.APP_PORT)): + with p.connect((self.master.options.onboarding_host, self.master.options.onbarding_port)): return p.request("get:'%s'" % page) else: p = self.pathoc() with p.connect(): - return p.request("get:'http://%s%s'" % (options.APP_HOST, page)) + return p.request("get:'http://%s%s'" % (self.master.options.onboarding_host, page)) class TransparentProxyTest(ProxyTestBase): From 52c8d7e0f8d86a096946f001f81cd938f3e54b88 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 15:00:40 +1300 Subject: [PATCH 06/12] addon options: shift server_replay options --- mitmproxy/addons/serverplayback.py | 56 +++++++++++++++++- mitmproxy/options.py | 61 -------------------- test/mitmproxy/addons/test_serverplayback.py | 32 +++++----- 3 files changed, 70 insertions(+), 79 deletions(-) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index d8b2299ad..73fb16666 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -1,8 +1,6 @@ import hashlib import urllib import typing -from typing import Any # noqa -from typing import List # noqa from mitmproxy import ctx from mitmproxy import flow @@ -19,6 +17,60 @@ class ServerPlayback: self.final_flow = None self.configured = False + def load(self, loader): + loader.add_option( + "server_replay_kill_extra", bool, False, + "Kill extra requests during replay." + ) + loader.add_option( + "server_replay_nopop", bool, False, + """ + Don't remove flows from server replay state after use. This makes it + possible to replay same response multiple times. + """ + ) + loader.add_option( + "server_replay_refresh", bool, True, + """ + Refresh server replay responses by adjusting date, expires and + last-modified headers, as well as adjusting cookie expiration. + """ + ) + loader.add_option( + "server_replay_use_headers", typing.Sequence[str], [], + "Request headers to be considered during replay." + ) + loader.add_option( + "server_replay", typing.Sequence[str], [], + "Replay server responses from a saved file." + ) + loader.add_option( + "server_replay_ignore_content", bool, False, + "Ignore request's content while searching for a saved flow to replay." + ) + loader.add_option( + "server_replay_ignore_params", typing.Sequence[str], [], + """ + Request's parameters to be ignored while searching for a saved flow + to replay. + """ + ) + loader.add_option( + "server_replay_ignore_payload_params", typing.Sequence[str], [], + """ + Request's payload parameters (application/x-www-form-urlencoded or + multipart/form-data) to be ignored while searching for a saved flow + to replay. + """ + ) + loader.add_option( + "server_replay_ignore_host", bool, False, + """ + Ignore request's destination host while searching for a saved flow + to replay. + """ + ) + @command.command("replay.server") def load_flows(self, flows: typing.Sequence[flow.Flow]) -> None: """ diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 318458581..02ae9c0fb 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -67,20 +67,11 @@ class Options(optmanager.OptManager): view_filter = None # type: Optional[str] # FIXME: Options that should be uncomplicated to migrate to addons - server_replay_refresh = None # type: bool replacements = None # type: Sequence[str] rfile = None # type: Optional[str] save_stream_file = None # type: Optional[str] save_stream_filter = None # type: Optional[str] scripts = None # type: Sequence[str] - server_replay = None # type: Sequence[str] - server_replay_ignore_content = None # type: bool - server_replay_ignore_host = None # type: bool - server_replay_ignore_params = None # type: Sequence[str] - server_replay_ignore_payload_params = None # type: Sequence[str] - server_replay_kill_extra = None # type: bool - server_replay_nopop = None # type: bool - server_replay_use_headers = None # type: Sequence[str] setheaders = None # type: Sequence[str] stickyauth = None # type: Optional[str] stickycookie = None # type: Optional[str] @@ -96,28 +87,10 @@ class Options(optmanager.OptManager): def __init__(self, **kwargs) -> None: super().__init__() - self.add_option( - "server_replay_kill_extra", bool, False, - "Kill extra requests during replay." - ) self.add_option( "server", bool, True, "Start a proxy server. Enabled by default." ) - self.add_option( - "server_replay_nopop", bool, False, - """ - Don't remove flows from server replay state after use. This makes it - possible to replay same response multiple times. - """ - ) - self.add_option( - "server_replay_refresh", bool, True, - """ - Refresh server replay responses by adjusting date, expires and - last-modified headers, as well as adjusting cookie expiration. - """ - ) self.add_option( "rfile", Optional[str], None, "Read flows from file." @@ -139,10 +112,6 @@ class Options(optmanager.OptManager): the separator can be any character. """ ) - self.add_option( - "server_replay_use_headers", Sequence[str], [], - "Request headers to be considered during replay." - ) self.add_option( "setheaders", Sequence[str], [], """ @@ -150,10 +119,6 @@ class Options(optmanager.OptManager): separator can be any character. """ ) - self.add_option( - "server_replay", Sequence[str], [], - "Replay server responses from a saved file." - ) self.add_option( "stickycookie", Optional[str], None, "Set sticky cookie filter. Matched against requests." @@ -195,32 +160,6 @@ class Options(optmanager.OptManager): "save_stream_filter", Optional[str], None, "Filter which flows are written to file." ) - self.add_option( - "server_replay_ignore_content", bool, False, - "Ignore request's content while searching for a saved flow to replay." - ) - self.add_option( - "server_replay_ignore_params", Sequence[str], [], - """ - Request's parameters to be ignored while searching for a saved flow - to replay. - """ - ) - self.add_option( - "server_replay_ignore_payload_params", Sequence[str], [], - """ - Request's payload parameters (application/x-www-form-urlencoded or - multipart/form-data) to be ignored while searching for a saved flow - to replay. - """ - ) - self.add_option( - "server_replay_ignore_host", bool, False, - """ - Ignore request's destination host while searching for a saved flow - to replay. - """ - ) # Proxy options self.add_option( diff --git a/test/mitmproxy/addons/test_serverplayback.py b/test/mitmproxy/addons/test_serverplayback.py index 32249a887..0bc28ac89 100644 --- a/test/mitmproxy/addons/test_serverplayback.py +++ b/test/mitmproxy/addons/test_serverplayback.py @@ -19,7 +19,7 @@ def tdump(path, flows): def test_load_file(tmpdir): s = serverplayback.ServerPlayback() - with taddons.context(): + with taddons.context(s): fpath = str(tmpdir.join("flows")) tdump(fpath, [tflow.tflow(resp=True)]) s.load_file(fpath) @@ -30,7 +30,7 @@ def test_load_file(tmpdir): def test_config(tmpdir): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: fpath = str(tmpdir.join("flows")) tdump(fpath, [tflow.tflow(resp=True)]) tctx.configure(s, server_replay=[fpath]) @@ -41,7 +41,7 @@ def test_config(tmpdir): def test_tick(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: s.stop = True s.final_flow = tflow.tflow() s.final_flow.live = False @@ -51,7 +51,7 @@ def test_tick(): def test_server_playback(): sp = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(sp) as tctx: tctx.configure(sp) f = tflow.tflow(resp=True) @@ -70,7 +70,7 @@ def test_server_playback(): def test_ignore_host(): sp = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(sp) as tctx: tctx.configure(sp, server_replay_ignore_host=True) r = tflow.tflow(resp=True) @@ -85,7 +85,7 @@ def test_ignore_host(): def test_ignore_content(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure(s, server_replay_ignore_content=False) r = tflow.tflow(resp=True) @@ -113,7 +113,7 @@ def test_ignore_content(): def test_ignore_content_wins_over_params(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure( s, server_replay_ignore_content=True, @@ -137,7 +137,7 @@ def test_ignore_content_wins_over_params(): def test_ignore_payload_params_other_content_type(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure( s, server_replay_ignore_content=False, @@ -161,7 +161,7 @@ def test_ignore_payload_params_other_content_type(): def test_hash(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure(s) r = tflow.tflow() @@ -181,7 +181,7 @@ def test_hash(): def test_headers(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure(s, server_replay_use_headers=["foo"]) r = tflow.tflow(resp=True) @@ -200,7 +200,7 @@ def test_headers(): def test_load(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure(s) r = tflow.tflow(resp=True) @@ -227,7 +227,7 @@ def test_load(): def test_load_with_server_replay_nopop(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure(s, server_replay_nopop=True) r = tflow.tflow(resp=True) @@ -245,7 +245,7 @@ def test_load_with_server_replay_nopop(): def test_ignore_params(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure( s, server_replay_ignore_params=["param1", "param2"] @@ -266,7 +266,7 @@ def test_ignore_params(): def thash(r, r2, setter): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: s = serverplayback.ServerPlayback() tctx.configure( s, @@ -328,7 +328,7 @@ def test_ignore_payload_params(): def test_server_playback_full(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure( s, server_replay_refresh = True, @@ -360,7 +360,7 @@ def test_server_playback_full(): def test_server_playback_kill(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure( s, server_replay_refresh = True, From 144b559b468ac2c67e0d79fae8754ae3385f1958 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 15:22:28 +1300 Subject: [PATCH 07/12] addon options: migrate replace, simplify taddons.context --- mitmproxy/addons/replace.py | 10 ++++++++++ mitmproxy/options.py | 8 -------- mitmproxy/test/taddons.py | 7 +++---- test/mitmproxy/addons/test_replace.py | 10 +++++----- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/mitmproxy/addons/replace.py b/mitmproxy/addons/replace.py index 054264fa4..5173254a3 100644 --- a/mitmproxy/addons/replace.py +++ b/mitmproxy/addons/replace.py @@ -1,5 +1,6 @@ import os import re +import typing from mitmproxy import exceptions from mitmproxy import flowfilter @@ -47,6 +48,15 @@ class Replace: def __init__(self): self.lst = [] + def load(self, loader): + loader.add_option( + "replacements", typing.Sequence[str], [], + """ + Replacement patterns of the form "/pattern/regex/replacement", where + the separator can be any character. + """ + ) + def configure(self, updated): """ .replacements is a list of tuples (fpat, rex, s): diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 02ae9c0fb..6db2b907a 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -67,7 +67,6 @@ class Options(optmanager.OptManager): view_filter = None # type: Optional[str] # FIXME: Options that should be uncomplicated to migrate to addons - replacements = None # type: Sequence[str] rfile = None # type: Optional[str] save_stream_file = None # type: Optional[str] save_stream_filter = None # type: Optional[str] @@ -105,13 +104,6 @@ class Options(optmanager.OptManager): "showhost", bool, False, "Use the Host header to construct URLs for display." ) - self.add_option( - "replacements", Sequence[str], [], - """ - Replacement patterns of the form "/pattern/regex/replacement", where - the separator can be any character. - """ - ) self.add_option( "setheaders", Sequence[str], [], """ diff --git a/mitmproxy/test/taddons.py b/mitmproxy/test/taddons.py index 12fc09861..7efa2eaf8 100644 --- a/mitmproxy/test/taddons.py +++ b/mitmproxy/test/taddons.py @@ -59,17 +59,16 @@ class context: provides a number of helper methods for common testing scenarios. """ - def __init__(self, *addons, master=None, options=None): + def __init__(self, *addons, options=None): options = options or mitmproxy.options.Options() - self.master = master or RecordingMaster( + self.master = RecordingMaster( options ) self.options = self.master.options self.wrapped = None - loader = addonmanager.Loader(self.master) for a in addons: - self.master.addons.invoke_addon(a, "load", loader) + self.master.addons.register(a) def ctx(self): """ diff --git a/test/mitmproxy/addons/test_replace.py b/test/mitmproxy/addons/test_replace.py index 9002afb5c..9c1f7f79f 100644 --- a/test/mitmproxy/addons/test_replace.py +++ b/test/mitmproxy/addons/test_replace.py @@ -18,7 +18,7 @@ class TestReplace: def test_configure(self): r = replace.Replace() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tctx.configure(r, replacements=["one/two/three"]) with pytest.raises(Exception, match="Invalid filter pattern"): tctx.configure(r, replacements=["/~b/two/three"]) @@ -28,7 +28,7 @@ class TestReplace: def test_simple(self): r = replace.Replace() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tctx.configure( r, replacements=[ @@ -48,7 +48,7 @@ class TestReplace: def test_order(self): r = replace.Replace() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tctx.configure( r, replacements=[ @@ -67,7 +67,7 @@ class TestReplace: class TestReplaceFile: def test_simple(self, tmpdir): r = replace.Replace() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tmpfile = tmpdir.join("replacement") tmpfile.write("bar") tctx.configure( @@ -81,7 +81,7 @@ class TestReplaceFile: def test_nonexistent(self, tmpdir): r = replace.Replace() - with taddons.context() as tctx: + with taddons.context(r) as tctx: with pytest.raises(Exception, match="Invalid file path"): tctx.configure( r, From 12633adeb9cbe3a81df2f6dfd9b739eae26bdcba Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 15:28:43 +1300 Subject: [PATCH 08/12] addon options: readfile, save, script --- mitmproxy/addons/readfile.py | 5 +++++ mitmproxy/addons/save.py | 10 ++++++++++ mitmproxy/addons/script.py | 8 ++++++++ mitmproxy/options.py | 22 ---------------------- test/mitmproxy/addons/test_readfile.py | 10 +++++----- 5 files changed, 28 insertions(+), 27 deletions(-) diff --git a/mitmproxy/addons/readfile.py b/mitmproxy/addons/readfile.py index 05b6c3099..aaf02d01b 100644 --- a/mitmproxy/addons/readfile.py +++ b/mitmproxy/addons/readfile.py @@ -11,6 +11,11 @@ class ReadFile: """ An addon that handles reading from file on startup. """ + def load(self, loader): + loader.add_option( + "rfile", typing.Optional[str], None, + "Read flows from file." + ) def load_flows(self, fo: typing.IO[bytes]) -> int: cnt = 0 diff --git a/mitmproxy/addons/save.py b/mitmproxy/addons/save.py index 44afef686..e6e98ff85 100644 --- a/mitmproxy/addons/save.py +++ b/mitmproxy/addons/save.py @@ -16,6 +16,16 @@ class Save: self.filt = None self.active_flows = set() # type: Set[flow.Flow] + def load(self, loader): + loader.add_option( + "save_stream_file", typing.Optional[str], None, + "Stream flows to file as they arrive. Prefix path with + to append." + ) + loader.add_option( + "save_stream_filter", typing.Optional[str], None, + "Filter which flows are written to file." + ) + def open_file(self, path): if path.startswith("+"): path = path[1:] diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 0a5243591..dcad943af 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -98,6 +98,14 @@ class ScriptLoader: self.is_running = False self.addons = [] + def load(self, loader): + loader.add_option( + "scripts", typing.Sequence[str], [], + """ + Execute a script. + """ + ) + def running(self): self.is_running = True diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 6db2b907a..e42ff0e9f 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -67,10 +67,6 @@ class Options(optmanager.OptManager): view_filter = None # type: Optional[str] # FIXME: Options that should be uncomplicated to migrate to addons - rfile = None # type: Optional[str] - save_stream_file = None # type: Optional[str] - save_stream_filter = None # type: Optional[str] - scripts = None # type: Sequence[str] setheaders = None # type: Sequence[str] stickyauth = None # type: Optional[str] stickycookie = None # type: Optional[str] @@ -90,16 +86,6 @@ class Options(optmanager.OptManager): "server", bool, True, "Start a proxy server. Enabled by default." ) - self.add_option( - "rfile", Optional[str], None, - "Read flows from file." - ) - self.add_option( - "scripts", Sequence[str], [], - """ - Execute a script. - """ - ) self.add_option( "showhost", bool, False, "Use the Host header to construct URLs for display." @@ -144,14 +130,6 @@ class Options(optmanager.OptManager): "The default content view mode.", choices = [i.name.lower() for i in contentviews.views] ) - self.add_option( - "save_stream_file", Optional[str], None, - "Stream flows to file as they arrive. Prefix path with + to append." - ) - self.add_option( - "save_stream_filter", Optional[str], None, - "Filter which flows are written to file." - ) # Proxy options self.add_option( diff --git a/test/mitmproxy/addons/test_readfile.py b/test/mitmproxy/addons/test_readfile.py index 813aa10e5..0439862ae 100644 --- a/test/mitmproxy/addons/test_readfile.py +++ b/test/mitmproxy/addons/test_readfile.py @@ -41,7 +41,7 @@ class TestReadFile: @mock.patch('mitmproxy.master.Master.load_flow') def test_configure(self, mck, tmpdir, data, corrupt_data): rf = readfile.ReadFile() - with taddons.context() as tctx: + with taddons.context(rf) as tctx: tf = tmpdir.join("tfile") tf.write(data.getvalue()) @@ -58,7 +58,7 @@ class TestReadFile: @mock.patch('mitmproxy.master.Master.load_flow') def test_corrupt(self, mck, corrupt_data): rf = readfile.ReadFile() - with taddons.context() as tctx: + with taddons.context(rf) as tctx: with pytest.raises(exceptions.FlowReadException): rf.load_flows(io.BytesIO(b"qibble")) assert not mck.called @@ -71,7 +71,7 @@ class TestReadFile: def test_nonexisting_file(self): rf = readfile.ReadFile() - with taddons.context() as tctx: + with taddons.context(rf) as tctx: with pytest.raises(exceptions.FlowReadException): rf.load_flows_from_path("nonexistent") assert len(tctx.master.logs) == 1 @@ -82,7 +82,7 @@ class TestReadFileStdin: @mock.patch('sys.stdin') def test_stdin(self, stdin, load_flow, data, corrupt_data): rf = readfile.ReadFileStdin() - with taddons.context() as tctx: + with taddons.context(rf) as tctx: stdin.buffer = data tctx.configure(rf, rfile="-") assert not load_flow.called @@ -97,7 +97,7 @@ class TestReadFileStdin: @mock.patch('mitmproxy.master.Master.load_flow') def test_normal(self, load_flow, tmpdir, data): rf = readfile.ReadFileStdin() - with taddons.context(): + with taddons.context(rf): tfile = tmpdir.join("tfile") tfile.write(data.getvalue()) rf.load_flows_from_path(str(tfile)) From 704c1db1b7aa25c4ee6371bec811e92409b92398 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 15:45:12 +1300 Subject: [PATCH 09/12] addon options: setheaders, stickyauth --- mitmproxy/addons/setheaders.py | 11 +++++++++++ mitmproxy/addons/stickyauth.py | 8 ++++++++ mitmproxy/options.py | 12 ------------ mitmproxy/test/taddons.py | 2 +- test/mitmproxy/addons/test_core.py | 19 +++++++------------ test/mitmproxy/addons/test_script.py | 11 +++++------ test/mitmproxy/addons/test_setheaders.py | 4 ++-- test/mitmproxy/addons/test_stickyauth.py | 4 ++-- 8 files changed, 36 insertions(+), 35 deletions(-) diff --git a/mitmproxy/addons/setheaders.py b/mitmproxy/addons/setheaders.py index d4d16e400..3517f70f3 100644 --- a/mitmproxy/addons/setheaders.py +++ b/mitmproxy/addons/setheaders.py @@ -1,3 +1,5 @@ +import typing + from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import ctx @@ -44,6 +46,15 @@ class SetHeaders: def __init__(self): self.lst = [] + def load(self, loader): + loader.add_option( + "setheaders", typing.Sequence[str], [], + """ + Header set pattern of the form "/pattern/header/value", where the + separator can be any character. + """ + ) + def configure(self, updated): if "setheaders" in updated: self.lst = [] diff --git a/mitmproxy/addons/stickyauth.py b/mitmproxy/addons/stickyauth.py index 24831d5ba..ab0599eef 100644 --- a/mitmproxy/addons/stickyauth.py +++ b/mitmproxy/addons/stickyauth.py @@ -1,3 +1,5 @@ +import typing + from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import ctx @@ -8,6 +10,12 @@ class StickyAuth: self.flt = None self.hosts = {} + def load(self, loader): + loader.add_option( + "stickyauth", typing.Optional[str], None, + "Set sticky auth filter. Matched against requests." + ) + def configure(self, updated): if "stickyauth" in updated: if ctx.options.stickyauth: diff --git a/mitmproxy/options.py b/mitmproxy/options.py index e42ff0e9f..bf2012d7a 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -67,7 +67,6 @@ class Options(optmanager.OptManager): view_filter = None # type: Optional[str] # FIXME: Options that should be uncomplicated to migrate to addons - setheaders = None # type: Sequence[str] stickyauth = None # type: Optional[str] stickycookie = None # type: Optional[str] stream_large_bodies = None # type: Optional[str] @@ -90,21 +89,10 @@ class Options(optmanager.OptManager): "showhost", bool, False, "Use the Host header to construct URLs for display." ) - self.add_option( - "setheaders", Sequence[str], [], - """ - Header set pattern of the form "/pattern/header/value", where the - separator can be any character. - """ - ) self.add_option( "stickycookie", Optional[str], None, "Set sticky cookie filter. Matched against requests." ) - self.add_option( - "stickyauth", Optional[str], None, - "Set sticky auth filter. Matched against requests." - ) self.add_option( "stream_large_bodies", Optional[str], None, """ diff --git a/mitmproxy/test/taddons.py b/mitmproxy/test/taddons.py index 7efa2eaf8..5930e4141 100644 --- a/mitmproxy/test/taddons.py +++ b/mitmproxy/test/taddons.py @@ -68,7 +68,7 @@ class context: self.wrapped = None for a in addons: - self.master.addons.register(a) + self.master.addons.add(a) def ctx(self): """ diff --git a/test/mitmproxy/addons/test_core.py b/test/mitmproxy/addons/test_core.py index 5c9a8a0d2..3f95bed4e 100644 --- a/test/mitmproxy/addons/test_core.py +++ b/test/mitmproxy/addons/test_core.py @@ -128,28 +128,23 @@ def test_options(tmpdir): p = str(tmpdir.join("path")) sa = core.Core() with taddons.context() as tctx: - tctx.options.stickycookie = "foo" - assert tctx.options.stickycookie == "foo" - sa.options_reset() - assert tctx.options.stickycookie is None - - tctx.options.stickycookie = "foo" - tctx.options.stickyauth = "bar" - sa.options_reset_one("stickycookie") - assert tctx.options.stickycookie is None - assert tctx.options.stickyauth == "bar" + tctx.options.listen_host = "foo" + assert tctx.options.listen_host == "foo" + sa.options_reset_one("listen_host") + assert tctx.options.listen_host != "foo" with pytest.raises(exceptions.CommandError): sa.options_reset_one("unknown") + tctx.options.listen_host = "foo" sa.options_save(p) with pytest.raises(exceptions.CommandError): sa.options_save("/") sa.options_reset() - assert tctx.options.stickyauth is None + assert tctx.options.listen_host == "" sa.options_load(p) - assert tctx.options.stickyauth == "bar" + assert tctx.options.listen_host == "foo" sa.options_load("/nonexistent") diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index 78a5be6cf..73641d24a 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -171,7 +171,7 @@ class TestScriptLoader: "mitmproxy/data/addonscripts/recorder/recorder.py" ) sc = script.ScriptLoader() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: sc.script_run([tflow.tflow(resp=True)], rp) debug = [i.msg for i in tctx.master.logs if i.level == "debug"] assert debug == [ @@ -183,7 +183,7 @@ class TestScriptLoader: def test_script_run_nonexistent(self): sc = script.ScriptLoader() - with taddons.context(): + with taddons.context(sc): with pytest.raises(exceptions.CommandError): sc.script_run([tflow.tflow(resp=True)], "/") @@ -208,8 +208,7 @@ class TestScriptLoader: def test_dupes(self): sc = script.ScriptLoader() - with taddons.context() as tctx: - tctx.master.addons.add(sc) + with taddons.context(sc) as tctx: with pytest.raises(exceptions.OptionsError): tctx.configure( sc, @@ -232,7 +231,7 @@ class TestScriptLoader: def test_load_err(self): sc = script.ScriptLoader() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, scripts=[ tutils.test_data.path("mitmproxy/data/addonscripts/load_error.py") ]) @@ -242,7 +241,7 @@ class TestScriptLoader: pass # this is expected and normally guarded. # on the next tick we should not fail however. tctx.invoke(sc, "tick") - assert len(tctx.master.addons) == 0 + assert len(tctx.master.addons) == 1 def test_order(self): rec = tutils.test_data.path("mitmproxy/data/addonscripts/recorder") diff --git a/test/mitmproxy/addons/test_setheaders.py b/test/mitmproxy/addons/test_setheaders.py index 3aaee7f40..2d5e9e3c8 100644 --- a/test/mitmproxy/addons/test_setheaders.py +++ b/test/mitmproxy/addons/test_setheaders.py @@ -19,14 +19,14 @@ class TestSetHeaders: def test_configure(self): sh = setheaders.SetHeaders() - with taddons.context() as tctx: + with taddons.context(sh) as tctx: with pytest.raises(Exception, match="Invalid setheader filter pattern"): tctx.configure(sh, setheaders = ["/~b/one/two"]) tctx.configure(sh, setheaders = ["/foo/bar/voing"]) def test_setheaders(self): sh = setheaders.SetHeaders() - with taddons.context() as tctx: + with taddons.context(sh) as tctx: tctx.configure( sh, setheaders = [ diff --git a/test/mitmproxy/addons/test_stickyauth.py b/test/mitmproxy/addons/test_stickyauth.py index ef7d07930..7b422fdd1 100644 --- a/test/mitmproxy/addons/test_stickyauth.py +++ b/test/mitmproxy/addons/test_stickyauth.py @@ -9,7 +9,7 @@ from mitmproxy import exceptions def test_configure(): r = stickyauth.StickyAuth() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tctx.configure(r, stickyauth="~s") with pytest.raises(exceptions.OptionsError): tctx.configure(r, stickyauth="~~") @@ -20,7 +20,7 @@ def test_configure(): def test_simple(): r = stickyauth.StickyAuth() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tctx.configure(r, stickyauth=".*") f = tflow.tflow(resp=True) f.request.headers["authorization"] = "foo" From 2aa7ac584b7884b8fd7622aec86b47d1e33a4a11 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 15:49:38 +1300 Subject: [PATCH 10/12] addon options: stickycookie, streambodies --- mitmproxy/addons/stickycookie.py | 6 ++++++ mitmproxy/addons/streambodies.py | 19 ++++++++++++++++++ mitmproxy/options.py | 23 ---------------------- test/mitmproxy/addons/test_stickycookie.py | 16 +++++++-------- test/mitmproxy/addons/test_streambodies.py | 2 +- 5 files changed, 34 insertions(+), 32 deletions(-) diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index e58e0a589..e5f85fa15 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -34,6 +34,12 @@ class StickyCookie: self.jar = collections.defaultdict(dict) # type: Dict[TOrigin, Dict[str, str]] self.flt = None # type: Optional[flowfilter.TFilter] + def load(self, loader): + loader.add_option( + "stickycookie", Optional[str], None, + "Set sticky cookie filter. Matched against requests." + ) + def configure(self, updated): if "stickycookie" in updated: if ctx.options.stickycookie: diff --git a/mitmproxy/addons/streambodies.py b/mitmproxy/addons/streambodies.py index c841075fb..6ca9918fc 100644 --- a/mitmproxy/addons/streambodies.py +++ b/mitmproxy/addons/streambodies.py @@ -1,3 +1,5 @@ +import typing + from mitmproxy.net.http import http1 from mitmproxy import exceptions from mitmproxy import ctx @@ -8,6 +10,23 @@ class StreamBodies: def __init__(self): self.max_size = None + def load(self, loader): + loader.add_option( + "stream_large_bodies", typing.Optional[str], None, + """ + Stream data to the client if response body exceeds the given + threshold. If streamed, the body will not be stored in any way. + Understands k/m/g suffixes, i.e. 3m for 3 megabytes. + """ + ) + loader.add_option( + "stream_websockets", bool, False, + """ + Stream WebSocket messages between client and server. + Messages are captured and cannot be modified. + """ + ) + def configure(self, updated): if "stream_large_bodies" in updated and ctx.options.stream_large_bodies: try: diff --git a/mitmproxy/options.py b/mitmproxy/options.py index bf2012d7a..ed17489a7 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -67,10 +67,6 @@ class Options(optmanager.OptManager): view_filter = None # type: Optional[str] # FIXME: Options that should be uncomplicated to migrate to addons - stickyauth = None # type: Optional[str] - stickycookie = None # type: Optional[str] - stream_large_bodies = None # type: Optional[str] - stream_websockets = None # type: bool upstream_auth = None # type: Optional[str] view_order = None # type: str view_order_reversed = None # type: bool @@ -89,25 +85,6 @@ class Options(optmanager.OptManager): "showhost", bool, False, "Use the Host header to construct URLs for display." ) - self.add_option( - "stickycookie", Optional[str], None, - "Set sticky cookie filter. Matched against requests." - ) - self.add_option( - "stream_large_bodies", Optional[str], None, - """ - Stream data to the client if response body exceeds the given - threshold. If streamed, the body will not be stored in any way. - Understands k/m/g suffixes, i.e. 3m for 3 megabytes. - """ - ) - self.add_option( - "stream_websockets", bool, False, - """ - Stream WebSocket messages between client and server. - Messages are captured and cannot be modified. - """ - ) self.add_option( "verbosity", str, 'info', "Log verbosity.", diff --git a/test/mitmproxy/addons/test_stickycookie.py b/test/mitmproxy/addons/test_stickycookie.py index f77d019d1..4493e9cc2 100644 --- a/test/mitmproxy/addons/test_stickycookie.py +++ b/test/mitmproxy/addons/test_stickycookie.py @@ -15,7 +15,7 @@ def test_domain_match(): class TestStickyCookie: def test_config(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: with pytest.raises(Exception, match="invalid filter"): tctx.configure(sc, stickycookie="~b") @@ -26,7 +26,7 @@ class TestStickyCookie: def test_simple(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") f = tflow.tflow(resp=True) f.response.headers["set-cookie"] = "foo=bar" @@ -50,7 +50,7 @@ class TestStickyCookie: def test_response(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " \ @@ -68,7 +68,7 @@ class TestStickyCookie: def test_response_multiple(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") # Test setting of multiple cookies @@ -82,7 +82,7 @@ class TestStickyCookie: def test_response_weird(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") # Test setting of weird cookie keys @@ -100,7 +100,7 @@ class TestStickyCookie: def test_response_overwrite(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") # Test overwriting of a cookie value @@ -115,7 +115,7 @@ class TestStickyCookie: def test_response_delete(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") # Test that a cookie is be deleted @@ -127,7 +127,7 @@ class TestStickyCookie: def test_request(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") f = self._response(sc, "SSID=mooo", "www.google.com") diff --git a/test/mitmproxy/addons/test_streambodies.py b/test/mitmproxy/addons/test_streambodies.py index 426ec9ae0..b980ea0b7 100644 --- a/test/mitmproxy/addons/test_streambodies.py +++ b/test/mitmproxy/addons/test_streambodies.py @@ -7,7 +7,7 @@ import pytest def test_simple(): sa = streambodies.StreamBodies() - with taddons.context() as tctx: + with taddons.context(sa) as tctx: with pytest.raises(exceptions.OptionsError): tctx.configure(sa, stream_large_bodies = "invalid") tctx.configure(sa, stream_large_bodies = "10") From f361ea491c4719c2a17d4abc0ef894116556a4c1 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 15:58:37 +1300 Subject: [PATCH 11/12] addon options: upstream_auth Also the last "easy" option to shift. --- mitmproxy/addons/upstream_auth.py | 10 ++++++++++ mitmproxy/options.py | 16 ---------------- test/mitmproxy/addons/test_upstream_auth.py | 4 ++-- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/mitmproxy/addons/upstream_auth.py b/mitmproxy/addons/upstream_auth.py index 685494c23..ea6af337a 100644 --- a/mitmproxy/addons/upstream_auth.py +++ b/mitmproxy/addons/upstream_auth.py @@ -1,4 +1,5 @@ import re +import typing import base64 from mitmproxy import exceptions @@ -28,6 +29,15 @@ class UpstreamAuth(): def __init__(self): self.auth = None + def load(self, loader): + loader.add_option( + "upstream_auth", typing.Optional[str], None, + """ + Add HTTP Basic authentication to upstream proxy and reverse proxy + requests. Format: username:password. + """ + ) + def configure(self, updated): # FIXME: We're doing this because our proxy core is terminally confused # at the moment. Ideally, we should be able to check if we're in diff --git a/mitmproxy/options.py b/mitmproxy/options.py index ed17489a7..70d454fd3 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -66,15 +66,6 @@ class Options(optmanager.OptManager): verbosity = None # type: str view_filter = None # type: Optional[str] - # FIXME: Options that should be uncomplicated to migrate to addons - upstream_auth = None # type: Optional[str] - view_order = None # type: str - view_order_reversed = None # type: bool - web_debug = None # type: bool - web_iface = None # type: str - web_open_browser = None # type: bool - web_port = None # type: int - def __init__(self, **kwargs) -> None: super().__init__() self.add_option( @@ -221,13 +212,6 @@ class Options(optmanager.OptManager): --upstream-bind-address to spoof a fixed source address. """ ) - self.add_option( - "upstream_auth", Optional[str], None, - """ - Add HTTP Basic authentication to upstream proxy and reverse proxy - requests. Format: username:password. - """ - ) self.add_option( "ssl_version_client", str, "secure", """ diff --git a/test/mitmproxy/addons/test_upstream_auth.py b/test/mitmproxy/addons/test_upstream_auth.py index c7342bb5b..53b342a27 100644 --- a/test/mitmproxy/addons/test_upstream_auth.py +++ b/test/mitmproxy/addons/test_upstream_auth.py @@ -9,7 +9,7 @@ from mitmproxy.addons import upstream_auth def test_configure(): up = upstream_auth.UpstreamAuth() - with taddons.context() as tctx: + with taddons.context(up) as tctx: tctx.configure(up, upstream_auth="test:test") assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:test") @@ -29,7 +29,7 @@ def test_configure(): def test_simple(): up = upstream_auth.UpstreamAuth() - with taddons.context() as tctx: + with taddons.context(up) as tctx: tctx.configure(up, upstream_auth="foo:bar") f = tflow.tflow() From 4fe83be63ca986754bcc9ab741c1fce3698a5053 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Feb 2018 16:06:11 +1300 Subject: [PATCH 12/12] Resolve some left-over cross-addon dependencies --- test/mitmproxy/tools/console/test_master.py | 4 ---- test/mitmproxy/tools/web/test_static_viewer.py | 5 +++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/test/mitmproxy/tools/console/test_master.py b/test/mitmproxy/tools/console/test_master.py index 6f46ce9ef..6ea61991d 100644 --- a/test/mitmproxy/tools/console/test_master.py +++ b/test/mitmproxy/tools/console/test_master.py @@ -7,10 +7,6 @@ from mitmproxy.tools import console from ... import tservers -def test_options(): - assert options.Options(server_replay_kill_extra=True) - - class TestMaster(tservers.MasterTest): def mkmaster(self, **opts): if "verbosity" not in opts: diff --git a/test/mitmproxy/tools/web/test_static_viewer.py b/test/mitmproxy/tools/web/test_static_viewer.py index cfe6cd7f1..dfc45bc2f 100644 --- a/test/mitmproxy/tools/web/test_static_viewer.py +++ b/test/mitmproxy/tools/web/test_static_viewer.py @@ -8,7 +8,7 @@ from mitmproxy import flowfilter from mitmproxy.tools.web.app import flow_to_json from mitmproxy.tools.web import static_viewer -from mitmproxy.addons import save +from mitmproxy.addons import save, readfile def test_save_static(tmpdir): @@ -59,8 +59,9 @@ def test_save_flows_content(ctx, tmpdir): def test_static_viewer(tmpdir): s = static_viewer.StaticViewer() + rf = readfile.ReadFile() sa = save.Save() - with taddons.context() as tctx: + with taddons.context(rf) as tctx: sa.save([tflow.tflow(resp=True)], str(tmpdir.join('foo'))) tctx.master.addons.add(s) tctx.configure(s, web_static_viewer=str(tmpdir), rfile=str(tmpdir.join('foo')))