Merge pull request #1548 from cortesi/opts

Make various replay-related options clearer

mitmproxy/builtins/serverplayback.py

@@ -36,12 +36,12 @@ class ServerPlayback(object):
         queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
 
         key = [str(r.port), str(r.scheme), str(r.method), str(path)]
-        if not self.options.replay_ignore_content:
+        if not self.options.server_replay_ignore_content:
             form_contents = r.urlencoded_form or r.multipart_form
-            if self.options.replay_ignore_payload_params and form_contents:
+            if self.options.server_replay_ignore_payload_params and form_contents:
                 params = [
                     strutils.always_bytes(i)
-                    for i in self.options.replay_ignore_payload_params
+                    for i in self.options.server_replay_ignore_payload_params
                 ]
                 for p in form_contents.items(multi=True):
                     if p[0] not in params:
@@ -49,11 +49,11 @@ class ServerPlayback(object):
             else:
                 key.append(str(r.raw_content))
 
-        if not self.options.replay_ignore_host:
+        if not self.options.server_replay_ignore_host:
             key.append(r.host)
 
         filtered = []
-        ignore_params = self.options.replay_ignore_params or []
+        ignore_params = self.options.server_replay_ignore_params or []
         for p in queriesArray:
             if p[0] not in ignore_params:
                 filtered.append(p)
@@ -61,9 +61,9 @@ class ServerPlayback(object):
             key.append(p[0])
             key.append(p[1])
 
-        if self.options.rheaders:
+        if self.options.server_replay_use_headers:
             headers = []
-            for i in self.options.rheaders:
+            for i in self.options.server_replay_use_headers:
                 v = r.headers.get(i)
                 headers.append((i, v))
             key.append(headers)
@@ -78,7 +78,7 @@ class ServerPlayback(object):
         """
         hsh = self._hash(request)
         if hsh in self.flowmap:
-            if self.options.nopop:
+            if self.options.server_replay_nopop:
                 return self.flowmap[hsh][0]
             else:
                 ret = self.flowmap[hsh].pop(0)
@@ -97,18 +97,6 @@ class ServerPlayback(object):
                     raise exceptions.OptionsError(str(e))
                 self.load(flows)
 
-        # FIXME: These options have to be renamed to something more sensible -
-        # prefixed with serverplayback_ where appropriate, and playback_ where
-        # they're shared with client playback.
-        #
-        # options.kill
-        # options.rheaders,
-        # options.nopop,
-        # options.replay_ignore_params,
-        # options.replay_ignore_content,
-        # options.replay_ignore_payload_params,
-        # options.replay_ignore_host
-
     def tick(self):
         if self.stop and not self.final_flow.live:
             ctx.master.shutdown()
@@ -125,7 +113,7 @@ class ServerPlayback(object):
                 if not self.flowmap and not self.options.keepserving:
                     self.final_flow = f
                     self.stop = True
-            elif self.options.kill:
+            elif self.options.replay_kill_extra:
                 ctx.log.warn(
                     "server_playback: killed non-replay request {}".format(
                         f.request.url

mitmproxy/cmdline.py

@@ -218,10 +218,10 @@ def get_common_options(args):
         anticache=args.anticache,
         anticomp=args.anticomp,
         client_replay=args.client_replay,
-        kill=args.kill,
+        replay_kill_extra=args.replay_kill_extra,
         no_server=args.no_server,
         refresh_server_playback=not args.norefresh,
-        rheaders=args.rheaders,
+        server_replay_use_headers=args.server_replay_use_headers,
         rfile=args.rfile,
         replacements=reps,
         setheaders=setheaders,
@@ -233,11 +233,11 @@ def get_common_options(args):
         showhost=args.showhost,
         outfile=args.outfile,
         verbosity=args.verbose,
-        nopop=args.nopop,
-        replay_ignore_content=args.replay_ignore_content,
-        replay_ignore_params=args.replay_ignore_params,
-        replay_ignore_payload_params=args.replay_ignore_payload_params,
-        replay_ignore_host=args.replay_ignore_host,
+        server_replay_nopop=args.server_replay_nopop,
+        server_replay_ignore_content=args.server_replay_ignore_content,
+        server_replay_ignore_params=args.server_replay_ignore_params,
+        server_replay_ignore_payload_params=args.server_replay_ignore_payload_params,
+        server_replay_ignore_host=args.server_replay_ignore_host,
 
         auth_nonanonymous = args.auth_nonanonymous,
         auth_singleuser = args.auth_singleuser,
@@ -594,13 +594,13 @@ def server_replay(parser):
         help="Replay server responses from a saved file."
     )
     group.add_argument(
-        "-k", "--kill",
-        action="store_true", dest="kill", default=False,
+        "-k", "--replay-kill-extra",
+        action="store_true", dest="replay_kill_extra", default=False,
         help="Kill extra requests during replay."
     )
     group.add_argument(
-        "--rheader",
-        action="append", dest="rheaders", type=str,
+        "--server-replay-use-header",
+        action="append", dest="server_replay_use_headers", type=str,
         help="Request headers to be considered during replay. "
              "Can be passed multiple times."
     )
@@ -614,21 +614,21 @@ def server_replay(parser):
     )
     group.add_argument(
         "--no-pop",
-        action="store_true", dest="nopop", default=False,
+        action="store_true", dest="server_replay_nopop", default=False,
         help="Disable response pop from response flow. "
              "This makes it possible to replay same response multiple times."
     )
     payload = group.add_mutually_exclusive_group()
     payload.add_argument(
         "--replay-ignore-content",
-        action="store_true", dest="replay_ignore_content", default=False,
+        action="store_true", dest="server_replay_ignore_content", default=False,
         help="""
             Ignore request's content while searching for a saved flow to replay
         """
     )
     payload.add_argument(
         "--replay-ignore-payload-param",
-        action="append", dest="replay_ignore_payload_params", type=str,
+        action="append", dest="server_replay_ignore_payload_params", type=str,
         help="""
             Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to
             be ignored while searching for a saved flow to replay.
@@ -638,7 +638,7 @@ def server_replay(parser):
 
     group.add_argument(
         "--replay-ignore-param",
-        action="append", dest="replay_ignore_params", type=str,
+        action="append", dest="server_replay_ignore_params", type=str,
         help="""
             Request's parameters to be ignored while searching for a saved flow
             to replay. Can be passed multiple times.
@@ -647,7 +647,7 @@ def server_replay(parser):
     group.add_argument(
         "--replay-ignore-host",
         action="store_true",
-        dest="replay_ignore_host",
+        dest="server_replay_ignore_host",
         default=False,
         help="Ignore request's destination host while searching for a saved flow to replay")
 

mitmproxy/console/flowlist.py

@@ -148,22 +148,22 @@ class ConnectionItem(urwid.WidgetWrap):
         if k == "a":
             self.master.start_server_playback(
                 [i.copy() for i in self.master.state.view],
-                self.master.options.kill, self.master.options.rheaders,
-                False, self.master.options.nopop,
-                self.master.options.replay_ignore_params,
-                self.master.options.replay_ignore_content,
-                self.master.options.replay_ignore_payload_params,
-                self.master.options.replay_ignore_host
+                self.master.options.replay_kill_extra, self.master.options.server_replay_use_headers,
+                False, self.master.options.server_replay_nopop,
+                self.master.options.server_replay_ignore_params,
+                self.master.options.server_replay_ignore_content,
+                self.master.options.server_replay_ignore_payload_params,
+                self.master.options.server_replay_ignore_host
             )
         elif k == "t":
             self.master.start_server_playback(
                 [self.flow.copy()],
-                self.master.options.kill, self.master.options.rheaders,
-                False, self.master.options.nopop,
-                self.master.options.replay_ignore_params,
-                self.master.options.replay_ignore_content,
-                self.master.options.replay_ignore_payload_params,
-                self.master.options.replay_ignore_host
+                self.master.options.replay_kill_extra, self.master.options.server_replay_use_headers,
+                False, self.master.options.server_replay_nopop,
+                self.master.options.server_replay_ignore_params,
+                self.master.options.server_replay_ignore_content,
+                self.master.options.server_replay_ignore_payload_params,
+                self.master.options.server_replay_ignore_host
             )
         else:
             signals.status_prompt_path.send(

mitmproxy/console/options.py

@@ -114,7 +114,7 @@ class Options(urwid.WidgetWrap):
                 select.Option(
                     "Kill Extra",
                     "x",
-                    lambda: master.options.kill,
+                    lambda: master.options.replay_kill_extra,
                     master.options.toggler("kill")
                 ),
                 select.Option(
@@ -165,7 +165,7 @@ class Options(urwid.WidgetWrap):
             anticomp = False,
             ignore_hosts = (),
             tcp_hosts = (),
-            kill = False,
+            replay_kill_extra = False,
             no_upstream_cert = False,
             refresh_server_playback = True,
             replacements = [],

mitmproxy/console/statusbar.py

@@ -191,7 +191,7 @@ class StatusBar(urwid.WidgetWrap):
             opts.append("showhost")
         if not self.master.options.refresh_server_playback:
             opts.append("norefresh")
-        if self.master.options.kill:
+        if self.master.options.replay_kill_extra:
             opts.append("killextra")
         if self.master.options.no_upstream_cert:
             opts.append("no-upstream-cert")

mitmproxy/dump.py

@@ -46,11 +46,11 @@ class DumpMaster(flow.FlowMaster):
         self.addons.add(options, dumper.Dumper())
         # This line is just for type hinting
         self.options = self.options  # type: Options
-        self.replay_ignore_params = options.replay_ignore_params
-        self.replay_ignore_content = options.replay_ignore_content
-        self.replay_ignore_host = options.replay_ignore_host
+        self.server_replay_ignore_params = options.server_replay_ignore_params
+        self.server_replay_ignore_content = options.server_replay_ignore_content
+        self.server_replay_ignore_host = options.server_replay_ignore_host
         self.refresh_server_playback = options.refresh_server_playback
-        self.replay_ignore_payload_params = options.replay_ignore_payload_params
+        self.server_replay_ignore_payload_params = options.server_replay_ignore_payload_params
 
         self.set_stream_large_bodies(options.stream_large_bodies)
 

mitmproxy/options.py

@@ -30,16 +30,16 @@ class Options(optmanager.OptManager):
             anticache=False,  # type: bool
             anticomp=False,  # type: bool
             client_replay=None,  # type: Optional[str]
-            kill=False,  # type: bool
+            replay_kill_extra=False,  # type: bool
             keepserving=True,  # type: bool
             no_server=False,  # type: bool
-            nopop=False,  # type: bool
+            server_replay_nopop=False,  # type: bool
             refresh_server_playback=False,  # type: bool
             rfile=None,  # type: Optional[str]
             scripts=(),  # type: Sequence[str]
             showhost=False,  # type: bool
             replacements=(),  # type: Sequence[Tuple[str, str, str]]
-            rheaders=(),  # type: Sequence[str]
+            server_replay_use_headers=(),  # type: Sequence[str]
             setheaders=(),  # type: Sequence[Tuple[str, str, str]]
             server_replay=None,  # type: Optional[str]
             stickycookie=None,  # type: Optional[str]
@@ -47,10 +47,10 @@ class Options(optmanager.OptManager):
             stream_large_bodies=None,  # type: Optional[str]
             verbosity=2,  # type: int
             outfile=None,  # type: Tuple[str, str]
-            replay_ignore_content=False,  # type: bool
-            replay_ignore_params=(),  # type: Sequence[str]
-            replay_ignore_payload_params=(),  # type: Sequence[str]
-            replay_ignore_host=False,  # type: bool
+            server_replay_ignore_content=False,  # type: bool
+            server_replay_ignore_params=(),  # type: Sequence[str]
+            server_replay_ignore_payload_params=(),  # type: Sequence[str]
+            server_replay_ignore_host=False,  # type: bool
 
             # Proxy options
             auth_nonanonymous=False,  # type: bool
@@ -89,15 +89,15 @@ class Options(optmanager.OptManager):
         self.anticomp = anticomp
         self.client_replay = client_replay
         self.keepserving = keepserving
-        self.kill = kill
+        self.replay_kill_extra = replay_kill_extra
         self.no_server = no_server
-        self.nopop = nopop
+        self.server_replay_nopop = server_replay_nopop
         self.refresh_server_playback = refresh_server_playback
         self.rfile = rfile
         self.scripts = scripts
         self.showhost = showhost
         self.replacements = replacements
-        self.rheaders = rheaders
+        self.server_replay_use_headers = server_replay_use_headers
         self.setheaders = setheaders
         self.server_replay = server_replay
         self.stickycookie = stickycookie
@@ -105,10 +105,10 @@ class Options(optmanager.OptManager):
         self.stream_large_bodies = stream_large_bodies
         self.verbosity = verbosity
         self.outfile = outfile
-        self.replay_ignore_content = replay_ignore_content
-        self.replay_ignore_params = replay_ignore_params
-        self.replay_ignore_payload_params = replay_ignore_payload_params
-        self.replay_ignore_host = replay_ignore_host
+        self.server_replay_ignore_content = server_replay_ignore_content
+        self.server_replay_ignore_params = server_replay_ignore_params
+        self.server_replay_ignore_payload_params = server_replay_ignore_payload_params
+        self.server_replay_ignore_host = server_replay_ignore_host
 
         # Proxy options
         self.auth_nonanonymous = auth_nonanonymous

test/mitmproxy/builtins/test_serverplayback.py

@@ -22,7 +22,7 @@ class TestServerPlayback:
 
     def test_ignore_host(self):
         sp = serverplayback.ServerPlayback()
-        sp.configure(options.Options(replay_ignore_host=True), [])
+        sp.configure(options.Options(server_replay_ignore_host=True), [])
 
         r = tutils.tflow(resp=True)
         r2 = tutils.tflow(resp=True)
@@ -35,7 +35,7 @@ class TestServerPlayback:
 
     def test_ignore_content(self):
         s = serverplayback.ServerPlayback()
-        s.configure(options.Options(replay_ignore_content=False), [])
+        s.configure(options.Options(server_replay_ignore_content=False), [])
 
         r = tutils.tflow(resp=True)
         r2 = tutils.tflow(resp=True)
@@ -46,7 +46,7 @@ class TestServerPlayback:
         r2.request.content = b"bar"
         assert not s._hash(r) == s._hash(r2)
 
-        s.configure(options.Options(replay_ignore_content=True), [])
+        s.configure(options.Options(server_replay_ignore_content=True), [])
         r = tutils.tflow(resp=True)
         r2 = tutils.tflow(resp=True)
         r.request.content = b"foo"
@@ -63,8 +63,8 @@ class TestServerPlayback:
         s = serverplayback.ServerPlayback()
         s.configure(
             options.Options(
-                replay_ignore_content=True,
-                replay_ignore_payload_params=[
+                server_replay_ignore_content=True,
+                server_replay_ignore_payload_params=[
                     "param1", "param2"
                 ]
             ),
@@ -87,8 +87,8 @@ class TestServerPlayback:
         s = serverplayback.ServerPlayback()
         s.configure(
             options.Options(
-                replay_ignore_content=False,
-                replay_ignore_payload_params=[
+                server_replay_ignore_content=False,
+                server_replay_ignore_payload_params=[
                     "param1", "param2"
                 ]
             ),
@@ -127,7 +127,7 @@ class TestServerPlayback:
 
     def test_headers(self):
         s = serverplayback.ServerPlayback()
-        s.configure(options.Options(rheaders=["foo"]), [])
+        s.configure(options.Options(server_replay_use_headers=["foo"]), [])
 
         r = tutils.tflow(resp=True)
         r.request.headers["foo"] = "bar"
@@ -167,9 +167,9 @@ class TestServerPlayback:
 
         assert not s.next_flow(r)
 
-    def test_load_with_nopop(self):
+    def test_load_with_server_replay_nopop(self):
         s = serverplayback.ServerPlayback()
-        s.configure(options.Options(nopop=True), [])
+        s.configure(options.Options(server_replay_nopop=True), [])
 
         r = tutils.tflow(resp=True)
         r.request.headers["key"] = "one"
@@ -187,7 +187,7 @@ class TestServerPlayback:
         s = serverplayback.ServerPlayback()
         s.configure(
             options.Options(
-                replay_ignore_params=["param1", "param2"]
+                server_replay_ignore_params=["param1", "param2"]
             ),
             []
         )
@@ -208,7 +208,7 @@ class TestServerPlayback:
         s = serverplayback.ServerPlayback()
         s.configure(
             options.Options(
-                replay_ignore_payload_params=["param1", "param2"]
+                server_replay_ignore_payload_params=["param1", "param2"]
             ),
             []
         )
@@ -270,7 +270,7 @@ class TestServerPlayback:
     def test_server_playback_kill(self):
         state = flow.State()
         s = serverplayback.ServerPlayback()
-        o = options.Options(refresh_server_playback = True, kill=True)
+        o = options.Options(refresh_server_playback = True, replay_kill_extra=True)
         m = mastertest.RecordingMaster(o, None, state)
         m.addons.add(o, s)
 

test/mitmproxy/console/test_master.py

@@ -107,7 +107,7 @@ def test_format_keyvals():
 
 
 def test_options():
-    assert console.master.Options(kill=True)
+    assert console.master.Options(replay_kill_extra=True)
 
 
 class TestMaster(mastertest.MasterTest):

test/mitmproxy/test_dump.py

@@ -49,14 +49,14 @@ class TestDumpMaster(mastertest.MasterTest):
         assert "error" in o.tfile.getvalue()
 
     def test_replay(self):
-        o = dump.Options(server_replay=["nonexistent"], kill=True)
+        o = dump.Options(server_replay=["nonexistent"], replay_kill_extra=True)
         tutils.raises(exceptions.OptionsError, dump.DumpMaster, None, o)
 
         with tutils.tmpdir() as t:
             p = os.path.join(t, "rep")
             self.flowfile(p)
 
-            o = dump.Options(server_replay=[p], kill=True)
+            o = dump.Options(server_replay=[p], replay_kill_extra=True)
             o.verbosity = 0
             o.flow_detail = 0
             m = dump.DumpMaster(None, o)
@@ -64,13 +64,13 @@ class TestDumpMaster(mastertest.MasterTest):
             self.cycle(m, b"content")
             self.cycle(m, b"content")
 
-            o = dump.Options(server_replay=[p], kill=False)
+            o = dump.Options(server_replay=[p], replay_kill_extra=False)
             o.verbosity = 0
             o.flow_detail = 0
             m = dump.DumpMaster(None, o)
             self.cycle(m, b"nonexistent")
 
-            o = dump.Options(client_replay=[p], kill=False)
+            o = dump.Options(client_replay=[p], replay_kill_extra=False)
             o.verbosity = 0
             o.flow_detail = 0
             m = dump.DumpMaster(None, o)