Merge pull request #432 from tekii/master

--replay_ignore_payload_params added, to filter params in form posts

libmproxy/cmdline.py

@@ -183,7 +183,8 @@ def get_common_options(options):
         verbosity=options.verbose,
         nopop=options.nopop,
         replay_ignore_content = options.replay_ignore_content,
-        replay_ignore_params = options.replay_ignore_params
+        replay_ignore_params = options.replay_ignore_params,
+        replay_ignore_payload_params = options.replay_ignore_payload_params
     )
 
 
@@ -438,13 +439,24 @@ def common_options(parser):
         help="Disable response pop from response flow. "
              "This makes it possible to replay same response multiple times."
     )
-    group.add_argument(
+    payload = group.add_mutually_exclusive_group()
+    payload.add_argument(
         "--replay-ignore-content",
         action="store_true", dest="replay_ignore_content", default=False,
         help="""
             Ignore request's content while searching for a saved flow to replay
         """
     )
+    payload.add_argument(
+        "--replay-ignore-payload-param",
+        action="append", dest="replay_ignore_payload_params", type=str,
+        help="""
+            Request's payload parameters (application/x-www-form-urlencoded) to 
+            be ignored while searching for a saved flow to replay. 
+            Can be passed multiple times. 
+        """
+    )
+
     group.add_argument(
         "--replay-ignore-param",
         action="append", dest="replay_ignore_params", type=str,

libmproxy/dump.py

@@ -39,6 +39,7 @@ class Options(object):
         "outfile",
         "replay_ignore_content",
         "replay_ignore_params",
+        "replay_ignore_payload_params",
     ]
 
     def __init__(self, **kwargs):
@@ -78,6 +79,7 @@ class DumpMaster(flow.FlowMaster):
         self.replay_ignore_params = options.replay_ignore_params
         self.replay_ignore_content = options.replay_ignore_content
         self.refresh_server_playback = options.refresh_server_playback
+        self.replay_ignore_payload_params = options.replay_ignore_payload_params
 
         self.set_stream_large_bodies(options.stream_large_bodies)
 
@@ -115,7 +117,8 @@ class DumpMaster(flow.FlowMaster):
                 not options.keepserving,
                 options.nopop,
                 options.replay_ignore_params,
-                options.replay_ignore_content
+                options.replay_ignore_content,
+                options.replay_ignore_payload_params,
             )
 
         if options.client_replay:

libmproxy/flow.py

@@ -201,12 +201,12 @@ class ClientPlaybackState:
 
 
 class ServerPlaybackState:
-    def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content):
+    def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content, ignore_payload_params):
         """
             headers: Case-insensitive list of request headers that should be
             included in request-response matching.
         """
-        self.headers, self.exit, self.nopop, self.ignore_params, self.ignore_content = headers, exit, nopop, ignore_params, ignore_content
+        self.headers, self.exit, self.nopop, self.ignore_params, self.ignore_content, self.ignore_payload_params = headers, exit, nopop, ignore_params, ignore_content, ignore_payload_params
         self.fmap = {}
         for i in flows:
             if i.response:
@@ -225,22 +225,37 @@ class ServerPlaybackState:
         _, _, path, _, query, _ = urlparse.urlparse(r.url)
         queriesArray = urlparse.parse_qsl(query)
 
-        filtered = []
-        ignore_params = self.ignore_params or []
-        for p in queriesArray:
-            if p[0] not in ignore_params:
-                filtered.append(p)
-
         key = [
             str(r.host),
             str(r.port),
             str(r.scheme),
             str(r.method),
             str(path),
-         ]
+        ]
-        if not self.ignore_content:
-            key.append(str(r.content))
 
+        if not self.ignore_content:
+            ignore_payload_params = self.ignore_payload_params or []
+            ct = r.headers["Content-Type"]
+            if len(ct) > 0:
+                ct = ct[0] 
+            if len(ignore_payload_params) > 0 and ct == "application/x-www-form-urlencoded": 
+                parsedContent = urlparse.parse_qsl(r.content)
+                filtered = []
+                for p in parsedContent:
+                    if p[0] not in ignore_payload_params:
+                        filtered.append(p)
+
+                for p in filtered:
+                    key.append(p[0])
+                    key.append(p[1])
+            else:
+                key.append(str(r.content))
+
+        filtered = []
+        ignore_params = self.ignore_params or []
+        for p in queriesArray:
+            if p[0] not in ignore_params:
+                filtered.append(p)
         for p in filtered:
             key.append(p[0])
             key.append(p[1])
@@ -697,14 +712,14 @@ class FlowMaster(controller.Master):
     def stop_client_playback(self):
         self.client_playback = None
 
-    def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params, ignore_content):
+    def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params, ignore_content, ignore_payload_params):
         """
             flows: List of flows.
             kill: Boolean, should we kill requests not part of the replay?
             ignore_params: list of parameters to ignore in server replay
             ignore_content: true if request content should be ignored in server replay
         """
-        self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, ignore_params, ignore_content)
+        self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, ignore_params, ignore_content, ignore_payload_params)
         self.kill_nonreplay = kill
 
     def stop_server_playback(self):

test/test_flow.py

@@ -114,7 +114,7 @@ class TestClientPlaybackState:
 
 class TestServerPlaybackState:
     def test_hash(self):
-        s = flow.ServerPlaybackState(None, [], False, False, None, False)
+        s = flow.ServerPlaybackState(None, [], False, False, None, False, None)
         r = tutils.tflow()
         r2 = tutils.tflow()
 
@@ -126,7 +126,7 @@ class TestServerPlaybackState:
         assert s._hash(r) != s._hash(r2)
 
     def test_headers(self):
-        s = flow.ServerPlaybackState(["foo"], [], False, False, None, False)
+        s = flow.ServerPlaybackState(["foo"], [], False, False, None, False, None)
         r = tutils.tflow(resp=True)
         r.request.headers["foo"] = ["bar"]
         r2 = tutils.tflow(resp=True)
@@ -147,7 +147,7 @@ class TestServerPlaybackState:
         r2 = tutils.tflow(resp=True)
         r2.request.headers["key"] = ["two"]
 
-        s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False)
+        s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False, None)
         assert s.count() == 2
         assert len(s.fmap.keys()) == 1
 
@@ -168,7 +168,7 @@ class TestServerPlaybackState:
         r2 = tutils.tflow(resp=True)
         r2.request.headers["key"] = ["two"]
 
-        s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False)
+        s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False, None)
 
         assert s.count() == 2
         s.next_flow(r)
@@ -176,7 +176,7 @@ class TestServerPlaybackState:
 
 
     def test_ignore_params(self):
-        s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False)
+        s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False, None)
         r = tutils.tflow(resp=True)
         r.request.path="/test?param1=1"
         r2 = tutils.tflow(resp=True)
@@ -189,8 +189,60 @@ class TestServerPlaybackState:
         r2.request.path="/test?param3=2"
         assert not s._hash(r) == s._hash(r2)
 
+    def test_ignore_payload_params(self):
+        s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"])
+        r = tutils.tflow(resp=True)
+        r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"]
+        r.request.content = "paramx=x&param1=1"
+        r2 = tutils.tflow(resp=True)
+        r2.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"]
+        r2.request.content = "paramx=x&param1=1"
+        # same parameters 
+        assert s._hash(r) == s._hash(r2)
+        # ignored parameters != 
+        r2.request.content = "paramx=x&param1=2"
+        assert s._hash(r) == s._hash(r2)
+        # missing parameter 
+        r2.request.content="paramx=x"
+        assert s._hash(r) == s._hash(r2)
+        # ignorable parameter added
+        r2.request.content="paramx=x&param1=2"
+        assert s._hash(r) == s._hash(r2)
+        # not ignorable parameter changed
+        r2.request.content="paramx=y&param1=1"
+        assert not s._hash(r) == s._hash(r2)
+        # not ignorable parameter missing
+        r2.request.content="param1=1"
+        assert not s._hash(r) == s._hash(r2)
+
+    def test_ignore_payload_params_other_content_type(self):
+        s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"])
+        r = tutils.tflow(resp=True)
+        r.request.headers["Content-Type"] = ["application/json"]
+        r.request.content = '{"param1":"1"}'
+        r2 = tutils.tflow(resp=True)
+        r2.request.headers["Content-Type"] = ["application/json"]
+        r2.request.content = '{"param1":"1"}'
+        # same content 
+        assert s._hash(r) == s._hash(r2)
+        # distint content (note only x-www-form-urlencoded payload is analysed)
+        r2.request.content = '{"param1":"2"}'
+        assert not s._hash(r) == s._hash(r2)
+
+    def test_ignore_payload_wins_over_params(self):
+        #NOTE: parameters are mutually exclusive in options
+        s = flow.ServerPlaybackState(None, [], False, False, None, True, ["param1", "param2"])
+        r = tutils.tflow(resp=True)
+        r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"]
+        r.request.content = "paramx=y"
+        r2 = tutils.tflow(resp=True)
+        r2.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"]
+        r2.request.content = "paramx=x"
+        # same parameters 
+        assert s._hash(r) == s._hash(r2)
+
     def test_ignore_content(self):
-        s = flow.ServerPlaybackState(None, [], False, False, None, False)
+        s = flow.ServerPlaybackState(None, [], False, False, None, False, None)
         r = tutils.tflow(resp=True)
         r2 = tutils.tflow(resp=True)
         
@@ -201,7 +253,7 @@ class TestServerPlaybackState:
         assert not s._hash(r) == s._hash(r2)
 
         #now ignoring content
-        s = flow.ServerPlaybackState(None, [], False, False, None, True)
+        s = flow.ServerPlaybackState(None, [], False, False, None, True, None)
         r = tutils.tflow(resp=True)
         r2 = tutils.tflow(resp=True)
         r.request.content = "foo"
@@ -695,7 +747,7 @@ class TestFlowMaster:
         pb = [tutils.tflow(resp=True), f]
 
         fm = flow.FlowMaster(DummyServer(ProxyConfig()), s)
-        assert not fm.start_server_playback(pb, False, [], False, False, None, False)
+        assert not fm.start_server_playback(pb, False, [], False, False, None, False, None)
         assert not fm.start_client_playback(pb, False)
         fm.client_playback.testing = True
 
@@ -718,16 +770,16 @@ class TestFlowMaster:
         fm.refresh_server_playback = True
         assert not fm.do_server_playback(tutils.tflow())
 
-        fm.start_server_playback(pb, False, [], False, False, None, False)
+        fm.start_server_playback(pb, False, [], False, False, None, False, None)
         assert fm.do_server_playback(tutils.tflow())
 
-        fm.start_server_playback(pb, False, [], True, False, None, False)
+        fm.start_server_playback(pb, False, [], True, False, None, False, None)
         r = tutils.tflow()
         r.request.content = "gibble"
         assert not fm.do_server_playback(r)
         assert fm.do_server_playback(tutils.tflow())
 
-        fm.start_server_playback(pb, False, [], True, False, None, False)
+        fm.start_server_playback(pb, False, [], True, False, None, False, None)
         q = Queue.Queue()
         fm.tick(q, 0)
         assert fm.should_exit.is_set()
@@ -742,7 +794,7 @@ class TestFlowMaster:
         pb = [f]
         fm = flow.FlowMaster(None, s)
         fm.refresh_server_playback = True
-        fm.start_server_playback(pb, True, [], False, False, None, False)
+        fm.start_server_playback(pb, True, [], False, False, None, False, None)
 
         f = tutils.tflow()
         f.request.host = "nonexistent"