From fbdce4b629fe0e626818aafe81dc247f746d70cd Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 3 Nov 2016 12:44:19 +1300 Subject: [PATCH 1/6] addons filstreamer: tests and bugfixes - Move tests to taddons - Fix a bug where options updates caused streaming to halt --- .gitignore | 1 + mitmproxy/addons/filestreamer.py | 34 +++++++++---------- test/mitmproxy/addons/test_filestreamer.py | 38 ++++++++-------------- 3 files changed, 31 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index e942ea940..def8186d2 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ bower_components *.map sslkeylogfile.log .tox/ +.vscode diff --git a/mitmproxy/addons/filestreamer.py b/mitmproxy/addons/filestreamer.py index 031b44ab6..92224d217 100644 --- a/mitmproxy/addons/filestreamer.py +++ b/mitmproxy/addons/filestreamer.py @@ -21,23 +21,23 @@ class FileStreamer: def configure(self, options, updated): # We're already streaming - stop the previous stream and restart - if self.stream: - self.done() - - if options.outfile: - flt = None - if options.get("filtstr"): - flt = flowfilter.parse(options.filtstr) - if not flt: - raise exceptions.OptionsError( - "Invalid filter specification: %s" % options.filtstr - ) - path, mode = options.outfile - if mode not in ("wb", "ab"): - raise exceptions.OptionsError("Invalid mode.") - err = self.start_stream_to_path(path, mode, flt) - if err: - raise exceptions.OptionsError(err) + if "outfile" in updated: + if self.stream: + self.done() + if options.outfile: + flt = None + if options.get("filtstr"): + flt = flowfilter.parse(options.filtstr) + if not flt: + raise exceptions.OptionsError( + "Invalid filter specification: %s" % options.filtstr + ) + path, mode = options.outfile + if mode not in ("wb", "ab"): + raise exceptions.OptionsError("Invalid mode.") + err = self.start_stream_to_path(path, mode, flt) + if err: + raise exceptions.OptionsError(err) def tcp_start(self, flow): if self.stream: diff --git a/test/mitmproxy/addons/test_filestreamer.py b/test/mitmproxy/addons/test_filestreamer.py index 28094c436..f86912fca 100644 --- a/test/mitmproxy/addons/test_filestreamer.py +++ b/test/mitmproxy/addons/test_filestreamer.py @@ -1,44 +1,32 @@ from mitmproxy.test import tflow from mitmproxy.test import tutils - -from .. import mastertest +from mitmproxy.test import taddons import os.path - from mitmproxy.addons import filestreamer -from mitmproxy import master from mitmproxy import io -from mitmproxy import options -from mitmproxy import proxy -class TestStream(mastertest.MasterTest): - def test_stream(self): +def test_stream(): + sa = filestreamer.FileStreamer() + with taddons.context() as tctx: with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") def r(): - r = io.FlowReader(open(p, "rb")) - return list(r.stream()) + x = io.FlowReader(open(p, "rb")) + return list(x.stream()) - o = options.Options( - outfile = (p, "wb") - ) - m = master.Master(o, proxy.DummyServer()) - sa = filestreamer.FileStreamer() + tctx.configure(sa, outfile=(p, "wb")) - m.addons.add(sa) f = tflow.tflow(resp=True) - m.request(f) - m.response(f) - m.addons.remove(sa) - + sa.request(f) + sa.response(f) + tctx.configure(sa, outfile=None) assert r()[0].response - m.options.outfile = (p, "ab") - - m.addons.add(sa) + tctx.configure(sa, outfile=(p, "ab")) f = tflow.tflow() - m.request(f) - m.addons.remove(sa) + sa.request(f) + tctx.configure(sa, outfile=None) assert not r()[1].response From 43a822198923215b78ae08cef7a63d1ad24e37c7 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 3 Nov 2016 14:41:30 +1300 Subject: [PATCH 2/6] addons: filestreamer test cov to 100% Also address a bug in taddons that caused options changes that raised errors to be retained rather than rolled back. --- .gitignore | 1 - mitmproxy/addons/filestreamer.py | 21 +++++---- mitmproxy/test/taddons.py | 18 ++++++-- test/mitmproxy/addons/test_filestreamer.py | 51 +++++++++++++++++++--- 4 files changed, 69 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index def8186d2..e942ea940 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,3 @@ bower_components *.map sslkeylogfile.log .tox/ -.vscode diff --git a/mitmproxy/addons/filestreamer.py b/mitmproxy/addons/filestreamer.py index 92224d217..cb6f37622 100644 --- a/mitmproxy/addons/filestreamer.py +++ b/mitmproxy/addons/filestreamer.py @@ -8,6 +8,7 @@ from mitmproxy import io class FileStreamer: def __init__(self): self.stream = None + self.filt = None self.active_flows = set() # type: Set[flow.Flow] def start_stream_to_path(self, path, mode, flt): @@ -15,29 +16,27 @@ class FileStreamer: try: f = open(path, mode) except IOError as v: - return str(v) + raise exceptions.OptionsError(str(v)) self.stream = io.FilteredFlowWriter(f, flt) self.active_flows = set() def configure(self, options, updated): # We're already streaming - stop the previous stream and restart + if "filtstr" in updated: + if options.get("filtstr"): + self.filt = flowfilter.parse(options.filtstr) + if not self.filt: + raise exceptions.OptionsError( + "Invalid filter specification: %s" % options.filtstr + ) if "outfile" in updated: if self.stream: self.done() if options.outfile: - flt = None - if options.get("filtstr"): - flt = flowfilter.parse(options.filtstr) - if not flt: - raise exceptions.OptionsError( - "Invalid filter specification: %s" % options.filtstr - ) path, mode = options.outfile if mode not in ("wb", "ab"): raise exceptions.OptionsError("Invalid mode.") - err = self.start_stream_to_path(path, mode, flt) - if err: - raise exceptions.OptionsError(err) + self.start_stream_to_path(path, mode, self.filt) def tcp_start(self, flow): if self.stream: diff --git a/mitmproxy/test/taddons.py b/mitmproxy/test/taddons.py index 3cba6762e..7804b90db 100644 --- a/mitmproxy/test/taddons.py +++ b/mitmproxy/test/taddons.py @@ -1,7 +1,10 @@ +import contextlib + import mitmproxy.master import mitmproxy.options from mitmproxy import proxy from mitmproxy import events +from mitmproxy import exceptions class RecordingMaster(mitmproxy.master.Master): @@ -36,6 +39,15 @@ class context: self.wrapped = None return False + @contextlib.contextmanager + def _rollback(self, opts, updates): + old = opts._opts.copy() + try: + yield + except exceptions.OptionsError as e: + opts.__dict__["_opts"] = old + raise + def cycle(self, addon, f): """ Cycles the flow through the events for the flow. Stops if a reply @@ -55,6 +67,6 @@ class context: Options object with the given keyword arguments, then calls the configure method on the addon with the updated value. """ - for k, v in kwargs.items(): - setattr(self.options, k, v) - addon.configure(self.options, kwargs.keys()) + with self._rollback(self.options, kwargs): + self.options.update(**kwargs) + addon.configure(self.options, kwargs.keys()) diff --git a/test/mitmproxy/addons/test_filestreamer.py b/test/mitmproxy/addons/test_filestreamer.py index f86912fca..658a0aa83 100644 --- a/test/mitmproxy/addons/test_filestreamer.py +++ b/test/mitmproxy/addons/test_filestreamer.py @@ -3,19 +3,55 @@ from mitmproxy.test import tutils from mitmproxy.test import taddons import os.path -from mitmproxy.addons import filestreamer from mitmproxy import io +from mitmproxy import exceptions +from mitmproxy.tools import dump +from mitmproxy.addons import filestreamer -def test_stream(): +def test_configure(): + sa = filestreamer.FileStreamer() + with taddons.context(options=dump.Options()) as tctx: + with tutils.tmpdir() as tdir: + p = os.path.join(tdir, "foo") + tutils.raises( + exceptions.OptionsError, + tctx.configure, sa, outfile=(tdir, "ab") + ) + tutils.raises( + "invalid filter", + tctx.configure, sa, outfile=(p, "ab"), filtstr="~~" + ) + tutils.raises( + "invalid mode", + tctx.configure, sa, outfile=(p, "xx") + ) + + +def rd(p): + x = io.FlowReader(open(p, "rb")) + return list(x.stream()) + + +def test_tcp(): sa = filestreamer.FileStreamer() with taddons.context() as tctx: with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") + tctx.configure(sa, outfile=(p, "wb")) - def r(): - x = io.FlowReader(open(p, "rb")) - return list(x.stream()) + tt = tflow.ttcpflow() + sa.tcp_start(tt) + sa.tcp_end(tt) + tctx.configure(sa, outfile=None) + assert rd(p) + + +def test_simple(): + sa = filestreamer.FileStreamer() + with taddons.context() as tctx: + with tutils.tmpdir() as tdir: + p = os.path.join(tdir, "foo") tctx.configure(sa, outfile=(p, "wb")) @@ -23,10 +59,11 @@ def test_stream(): sa.request(f) sa.response(f) tctx.configure(sa, outfile=None) - assert r()[0].response + assert rd(p)[0].response tctx.configure(sa, outfile=(p, "ab")) f = tflow.tflow() sa.request(f) tctx.configure(sa, outfile=None) - assert not r()[1].response + assert not rd(p)[1].response + From 9f77c79227f424f48ec16871d7efd451c8de6d47 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 3 Nov 2016 15:00:18 +1300 Subject: [PATCH 3/6] FileStreamer -> StreamFile options.wfile -> options.streamfile --- mitmproxy/addons/__init__.py | 4 +-- .../addons/{filestreamer.py => streamfile.py} | 8 +++--- mitmproxy/options.py | 4 +-- mitmproxy/tools/cmdline.py | 16 +++++------ mitmproxy/tools/console/flowlist.py | 6 ++--- mitmproxy/tools/console/statusbar.py | 4 +-- mitmproxy/tools/web/master.py | 6 ++--- ...est_filestreamer.py => test_streamfile.py} | 27 +++++++++---------- 8 files changed, 37 insertions(+), 38 deletions(-) rename mitmproxy/addons/{filestreamer.py => streamfile.py} (93%) rename test/mitmproxy/addons/{test_filestreamer.py => test_streamfile.py} (67%) diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index c5d405253..d2b50c35e 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -1,7 +1,7 @@ from mitmproxy.addons import anticache from mitmproxy.addons import anticomp from mitmproxy.addons import clientplayback -from mitmproxy.addons import filestreamer +from mitmproxy.addons import streamfile from mitmproxy.addons import onboarding from mitmproxy.addons import replace from mitmproxy.addons import script @@ -20,7 +20,7 @@ def default_addons(): stickyauth.StickyAuth(), stickycookie.StickyCookie(), script.ScriptLoader(), - filestreamer.FileStreamer(), + streamfile.StreamFile(), streambodies.StreamBodies(), replace.Replace(), setheaders.SetHeaders(), diff --git a/mitmproxy/addons/filestreamer.py b/mitmproxy/addons/streamfile.py similarity index 93% rename from mitmproxy/addons/filestreamer.py rename to mitmproxy/addons/streamfile.py index cb6f37622..76befeb2b 100644 --- a/mitmproxy/addons/filestreamer.py +++ b/mitmproxy/addons/streamfile.py @@ -5,7 +5,7 @@ from mitmproxy import flowfilter from mitmproxy import io -class FileStreamer: +class StreamFile: def __init__(self): self.stream = None self.filt = None @@ -29,11 +29,11 @@ class FileStreamer: raise exceptions.OptionsError( "Invalid filter specification: %s" % options.filtstr ) - if "outfile" in updated: + if "streamfile" in updated: if self.stream: self.done() - if options.outfile: - path, mode = options.outfile + if options.streamfile: + path, mode = options.streamfile if mode not in ("wb", "ab"): raise exceptions.OptionsError("Invalid mode.") self.start_stream_to_path(path, mode, self.filt) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 1db9f0f08..84aa4a742 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -48,7 +48,7 @@ class Options(optmanager.OptManager): stream_large_bodies: Optional[int] = None, verbosity: int = 2, default_contentview: str = "auto", - outfile: Optional[Tuple[str, str]] = None, + streamfile: Optional[Tuple[str, str]] = None, server_replay_ignore_content: bool = False, server_replay_ignore_params: Sequence[str] = (), server_replay_ignore_payload_params: Sequence[str] = (), @@ -108,7 +108,7 @@ class Options(optmanager.OptManager): self.stream_large_bodies = stream_large_bodies self.verbosity = verbosity self.default_contentview = default_contentview - self.outfile = outfile + self.streamfile = streamfile self.server_replay_ignore_content = server_replay_ignore_content self.server_replay_ignore_params = server_replay_ignore_params self.server_replay_ignore_payload_params = server_replay_ignore_payload_params diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index e4b29d0fa..cb26378d9 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -140,8 +140,8 @@ def get_common_options(args): raise exceptions.OptionsError(e) setheaders.append(p) - if args.outfile and args.outfile[0] == args.rfile: - if args.outfile[1] == "wb": + if args.streamfile and args.streamfile[0] == args.rfile: + if args.streamfile[1] == "wb": raise exceptions.OptionsError( "Cannot use '{}' for both reading and writing flows. " "Are you looking for --afile?".format(args.rfile) @@ -228,7 +228,7 @@ def get_common_options(args): stickyauth=stickyauth, stream_large_bodies=stream_large_bodies, showhost=args.showhost, - outfile=args.outfile, + streamfile=args.streamfile, verbosity=args.verbose, server_replay_nopop=args.server_replay_nopop, server_replay_ignore_content=args.server_replay_ignore_content, @@ -339,15 +339,15 @@ def basic_options(parser): action="store_const", dest="verbose", default=2, const=3, help="Increase log verbosity." ) - outfile = parser.add_mutually_exclusive_group() - outfile.add_argument( + streamfile = parser.add_mutually_exclusive_group() + streamfile.add_argument( "-w", "--wfile", - action="store", dest="outfile", type=lambda f: (f, "wb"), + action="store", dest="streamfile", type=lambda f: (f, "wb"), help="Write flows to file." ) - outfile.add_argument( + streamfile.add_argument( "-a", "--afile", - action="store", dest="outfile", type=lambda f: (f, "ab"), + action="store", dest="streamfile", type=lambda f: (f, "ab"), help="Append flows to file." ) parser.add_argument( diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index 76545893c..fc8368e7d 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -393,13 +393,13 @@ class FlowListBox(urwid.ListBox): val = not self.master.options.order_reversed self.master.options.order_reversed = val elif key == "W": - if self.master.options.outfile: - self.master.options.outfile = None + if self.master.options.streamfile: + self.master.options.streamfile = None else: signals.status_prompt_path.send( self, prompt="Stream flows to", - callback= lambda path: self.master.options.update(outfile=(path, "ab")) + callback= lambda path: self.master.options.update(streamfile=(path, "ab")) ) else: return urwid.ListBox.keypress(self, size, key) diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index e292cbd79..ebf2bca8c 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -223,8 +223,8 @@ class StatusBar(urwid.WidgetWrap): r.append(("heading_key", "s")) r.append("cripts:%s]" % len(self.master.options.scripts)) - if self.master.options.outfile: - r.append("[W:%s]" % self.master.options.outfile[0]) + if self.master.options.streamfile: + r.append("[W:%s]" % self.master.options.streamfile[0]) return r diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index d283e5d5e..3e1b59339 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -111,10 +111,10 @@ class WebMaster(master.Master): "error" ) - if options.outfile: + if options.streamfile: err = self.start_stream_to_path( - options.outfile[0], - options.outfile[1] + options.streamfile[0], + options.streamfile[1] ) if err: print("Stream file error: {}".format(err), file=sys.stderr) diff --git a/test/mitmproxy/addons/test_filestreamer.py b/test/mitmproxy/addons/test_streamfile.py similarity index 67% rename from test/mitmproxy/addons/test_filestreamer.py rename to test/mitmproxy/addons/test_streamfile.py index 658a0aa83..f63f29c42 100644 --- a/test/mitmproxy/addons/test_filestreamer.py +++ b/test/mitmproxy/addons/test_streamfile.py @@ -6,25 +6,25 @@ import os.path from mitmproxy import io from mitmproxy import exceptions from mitmproxy.tools import dump -from mitmproxy.addons import filestreamer +from mitmproxy.addons import streamfile def test_configure(): - sa = filestreamer.FileStreamer() + sa = streamfile.StreamFile() with taddons.context(options=dump.Options()) as tctx: with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") tutils.raises( exceptions.OptionsError, - tctx.configure, sa, outfile=(tdir, "ab") + tctx.configure, sa, streamfile=(tdir, "ab") ) tutils.raises( "invalid filter", - tctx.configure, sa, outfile=(p, "ab"), filtstr="~~" + tctx.configure, sa, streamfile=(p, "ab"), filtstr="~~" ) tutils.raises( "invalid mode", - tctx.configure, sa, outfile=(p, "xx") + tctx.configure, sa, streamfile=(p, "xx") ) @@ -34,36 +34,35 @@ def rd(p): def test_tcp(): - sa = filestreamer.FileStreamer() + sa = streamfile.StreamFile() with taddons.context() as tctx: with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") - tctx.configure(sa, outfile=(p, "wb")) + tctx.configure(sa, streamfile=(p, "wb")) tt = tflow.ttcpflow() sa.tcp_start(tt) sa.tcp_end(tt) - tctx.configure(sa, outfile=None) + tctx.configure(sa, streamfile=None) assert rd(p) def test_simple(): - sa = filestreamer.FileStreamer() + sa = streamfile.StreamFile() with taddons.context() as tctx: with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") - tctx.configure(sa, outfile=(p, "wb")) + tctx.configure(sa, streamfile=(p, "wb")) f = tflow.tflow(resp=True) sa.request(f) sa.response(f) - tctx.configure(sa, outfile=None) + tctx.configure(sa, streamfile=None) assert rd(p)[0].response - tctx.configure(sa, outfile=(p, "ab")) + tctx.configure(sa, streamfile=(p, "ab")) f = tflow.tflow() sa.request(f) - tctx.configure(sa, outfile=None) + tctx.configure(sa, streamfile=None) assert not rd(p)[1].response - From 77e6dfe35ce26b91a289490abad41b903d102ba1 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 3 Nov 2016 15:12:04 +1300 Subject: [PATCH 4/6] streamfile: make options less crazy Instead of a (file, mode) tuple, move to "streamfile" as the path, and "streamfile_append" to specify if we should append or not. Leave the command-line options intact for now. --- mitmproxy/addons/streamfile.py | 9 ++++---- mitmproxy/options.py | 4 +++- mitmproxy/tools/cmdline.py | 7 +++--- mitmproxy/tools/console/flowlist.py | 2 +- mitmproxy/tools/console/statusbar.py | 2 +- mitmproxy/tools/web/master.py | 9 -------- test/mitmproxy/addons/test_streamfile.py | 14 +++++------- test/mitmproxy/test_dump.py | 28 ------------------------ 8 files changed, 19 insertions(+), 56 deletions(-) diff --git a/mitmproxy/addons/streamfile.py b/mitmproxy/addons/streamfile.py index 76befeb2b..377f277de 100644 --- a/mitmproxy/addons/streamfile.py +++ b/mitmproxy/addons/streamfile.py @@ -33,10 +33,11 @@ class StreamFile: if self.stream: self.done() if options.streamfile: - path, mode = options.streamfile - if mode not in ("wb", "ab"): - raise exceptions.OptionsError("Invalid mode.") - self.start_stream_to_path(path, mode, self.filt) + if options.streamfile_append: + mode = "ab" + else: + mode = "wb" + self.start_stream_to_path(options.streamfile, mode, self.filt) def tcp_start(self, flow): if self.stream: diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 84aa4a742..497914006 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -48,7 +48,8 @@ class Options(optmanager.OptManager): stream_large_bodies: Optional[int] = None, verbosity: int = 2, default_contentview: str = "auto", - streamfile: Optional[Tuple[str, str]] = None, + streamfile: Optional[str] = None, + streamfile_append: bool = False, server_replay_ignore_content: bool = False, server_replay_ignore_params: Sequence[str] = (), server_replay_ignore_payload_params: Sequence[str] = (), @@ -109,6 +110,7 @@ class Options(optmanager.OptManager): self.verbosity = verbosity self.default_contentview = default_contentview self.streamfile = streamfile + self.streamfile_append = streamfile_append self.server_replay_ignore_content = server_replay_ignore_content self.server_replay_ignore_params = server_replay_ignore_params self.server_replay_ignore_payload_params = server_replay_ignore_payload_params diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index cb26378d9..debe6db9f 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -228,7 +228,8 @@ def get_common_options(args): stickyauth=stickyauth, stream_large_bodies=stream_large_bodies, showhost=args.showhost, - streamfile=args.streamfile, + streamfile=args.streamfile[0] if args.streamfile else None, + streamfile_append=True if args.streamfile and args.streamfile[1] == "a" else False, verbosity=args.verbose, server_replay_nopop=args.server_replay_nopop, server_replay_ignore_content=args.server_replay_ignore_content, @@ -342,12 +343,12 @@ def basic_options(parser): streamfile = parser.add_mutually_exclusive_group() streamfile.add_argument( "-w", "--wfile", - action="store", dest="streamfile", type=lambda f: (f, "wb"), + action="store", dest="streamfile", type=lambda f: (f, "w"), help="Write flows to file." ) streamfile.add_argument( "-a", "--afile", - action="store", dest="streamfile", type=lambda f: (f, "ab"), + action="store", dest="streamfile", type=lambda f: (f, "a"), help="Append flows to file." ) parser.add_argument( diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index fc8368e7d..6ad7f6562 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -399,7 +399,7 @@ class FlowListBox(urwid.ListBox): signals.status_prompt_path.send( self, prompt="Stream flows to", - callback= lambda path: self.master.options.update(streamfile=(path, "ab")) + callback= lambda path: self.master.options.update(streamfile=path) ) else: return urwid.ListBox.keypress(self, size, key) diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index ebf2bca8c..e34244937 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -224,7 +224,7 @@ class StatusBar(urwid.WidgetWrap): r.append("cripts:%s]" % len(self.master.options.scripts)) if self.master.options.streamfile: - r.append("[W:%s]" % self.master.options.streamfile[0]) + r.append("[W:%s]" % self.master.options.streamfile) return r diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 3e1b59339..2f1fd4e39 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -111,15 +111,6 @@ class WebMaster(master.Master): "error" ) - if options.streamfile: - err = self.start_stream_to_path( - options.streamfile[0], - options.streamfile[1] - ) - if err: - print("Stream file error: {}".format(err), file=sys.stderr) - sys.exit(1) - def _sig_add(self, view, flow): app.ClientConnection.broadcast( type="UPDATE_FLOWS", diff --git a/test/mitmproxy/addons/test_streamfile.py b/test/mitmproxy/addons/test_streamfile.py index f63f29c42..82a4345b4 100644 --- a/test/mitmproxy/addons/test_streamfile.py +++ b/test/mitmproxy/addons/test_streamfile.py @@ -16,15 +16,11 @@ def test_configure(): p = os.path.join(tdir, "foo") tutils.raises( exceptions.OptionsError, - tctx.configure, sa, streamfile=(tdir, "ab") + tctx.configure, sa, streamfile=tdir ) tutils.raises( "invalid filter", - tctx.configure, sa, streamfile=(p, "ab"), filtstr="~~" - ) - tutils.raises( - "invalid mode", - tctx.configure, sa, streamfile=(p, "xx") + tctx.configure, sa, streamfile=p, filtstr="~~" ) @@ -38,7 +34,7 @@ def test_tcp(): with taddons.context() as tctx: with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") - tctx.configure(sa, streamfile=(p, "wb")) + tctx.configure(sa, streamfile=p) tt = tflow.ttcpflow() sa.tcp_start(tt) @@ -53,7 +49,7 @@ def test_simple(): with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") - tctx.configure(sa, streamfile=(p, "wb")) + tctx.configure(sa, streamfile=p) f = tflow.tflow(resp=True) sa.request(f) @@ -61,7 +57,7 @@ def test_simple(): tctx.configure(sa, streamfile=None) assert rd(p)[0].response - tctx.configure(sa, streamfile=(p, "ab")) + tctx.configure(sa, streamfile=p, streamfile_append=True) f = tflow.tflow() sa.request(f) tctx.configure(sa, streamfile=None) diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index aa3228e4e..e331637d9 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -2,7 +2,6 @@ from mitmproxy.test import tflow import os import io -import mitmproxy.io from mitmproxy.tools import dump from mitmproxy import exceptions from mitmproxy import proxy @@ -126,33 +125,6 @@ class TestDumpMaster(mastertest.MasterTest): f = self.cycle(m, b"content") assert f.request.headers["one"] == "two" - def test_write(self): - with tutils.tmpdir() as d: - p = os.path.join(d, "a") - self.dummy_cycle( - self.mkmaster(None, outfile=(p, "wb"), verbosity=0), 1, b"" - ) - assert len(list(mitmproxy.io.FlowReader(open(p, "rb")).stream())) == 1 - - def test_write_append(self): - with tutils.tmpdir() as d: - p = os.path.join(d, "a.append") - self.dummy_cycle( - self.mkmaster(None, outfile=(p, "wb"), verbosity=0), - 1, b"" - ) - self.dummy_cycle( - self.mkmaster(None, outfile=(p, "ab"), verbosity=0), - 1, b"" - ) - assert len(list(mitmproxy.io.FlowReader(open(p, "rb")).stream())) == 2 - - def test_write_err(self): - tutils.raises( - exceptions.OptionsError, - self.mkmaster, None, outfile = ("nonexistentdir/foo", "wb") - ) - def test_script(self): ret = self.dummy_cycle( self.mkmaster( From 9fc6674151a377c467369f3989800c5ab910edb8 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 3 Nov 2016 16:16:29 +1300 Subject: [PATCH 5/6] addons.intercept: 100% test coverage --- test/mitmproxy/addons/test_intercept.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/mitmproxy/addons/test_intercept.py b/test/mitmproxy/addons/test_intercept.py index efdac3744..5178cb972 100644 --- a/test/mitmproxy/addons/test_intercept.py +++ b/test/mitmproxy/addons/test_intercept.py @@ -36,3 +36,8 @@ def test_simple(): f = tflow.tflow(resp=False) tctx.cycle(r, f) assert not f.intercepted + + f = tflow.tflow(resp=True) + f.reply._state = "handled" + r.response(f) + assert f.intercepted \ No newline at end of file From d31f2698a5c2932eef7593f01ca80b8ddb21e18b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 3 Nov 2016 16:19:18 +1300 Subject: [PATCH 6/6] addons.onboarding: remove duplicate code --- mitmproxy/addons/onboardingapp/app.py | 16 ---------------- test/mitmproxy/addons/test_intercept.py | 2 +- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/mitmproxy/addons/onboardingapp/app.py b/mitmproxy/addons/onboardingapp/app.py index 50b52214a..d418952c7 100644 --- a/mitmproxy/addons/onboardingapp/app.py +++ b/mitmproxy/addons/onboardingapp/app.py @@ -6,7 +6,6 @@ import tornado.wsgi from mitmproxy.utils import data from mitmproxy.proxy import config -from mitmproxy.addons import wsgiapp loader = tornado.template.Loader(data.pkg_data.path("addons/onboardingapp/templates")) @@ -92,18 +91,3 @@ application = tornado.web.Application( ], # debug=True ) - - -class Onboarding(wsgiapp.WSGIApp): - def __init__(self): - super().__init__(Adapter(application), None, None) - self.enabled = False - - def configure(self, options, updated): - self.host = options.app_host - self.port = options.app_port - self.enabled = options.app - - def request(self, f): - if self.enabled: - super().request(f) diff --git a/test/mitmproxy/addons/test_intercept.py b/test/mitmproxy/addons/test_intercept.py index 5178cb972..a347f9abd 100644 --- a/test/mitmproxy/addons/test_intercept.py +++ b/test/mitmproxy/addons/test_intercept.py @@ -40,4 +40,4 @@ def test_simple(): f = tflow.tflow(resp=True) f.reply._state = "handled" r.response(f) - assert f.intercepted \ No newline at end of file + assert f.intercepted