Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-23 08:11:00 +00:00
Code Issues Projects Releases Wiki Activity

Adding a server replay option to ignore host when searching for matching requests

Browse Source
This commit is contained in:
Wade Catron 2015-03-07 08:38:18 -08:00
parent 15a8a93a4e
commit 803d631f04
6 changed files with 65 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libmproxy/cmdline.py
View File

@ -192,7 +192,8 @@ def get_common_options(options):
nopop=options.nopop, nopop=options.nopop,
replay_ignore_content = options.replay_ignore_content, replay_ignore_content = options.replay_ignore_content,
replay_ignore_params = options.replay_ignore_params, replay_ignore_params = options.replay_ignore_params,
replay_ignore_payload_params = options.replay_ignore_payload_params replay_ignore_payload_params = options.replay_ignore_payload_params,
replay_ignore_host = options.replay_ignore_host
) )
@ -479,6 +480,11 @@ def common_options(parser):
to replay. Can be passed multiple times. to replay. Can be passed multiple times.
""" """
) )
group.add_argument(
"--replay-ignore-host",
action="store_true", dest="replay_ignore_host", default=False,
help="Ignore request's destination host while searching for a saved flow to replay"
)
group = parser.add_argument_group( group = parser.add_argument_group(
"Replacements", "Replacements",

5
libmproxy/console/__init__.py
View File

@ -528,7 +528,10 @@ class ConsoleMaster(flow.FlowMaster):
flows, flows,
self.killextra, self.rheaders, self.killextra, self.rheaders,
False, self.nopop, False, self.nopop,
self.options.replay_ignore_params, self.options.replay_ignore_content, self.options.replay_ignore_payload_params self.options.replay_ignore_params,
self.options.replay_ignore_content,
self.options.replay_ignore_payload_params,
self.options.replay_ignore_host
) )
def spawn_editor(self, data): def spawn_editor(self, data):

6
libmproxy/console/flowlist.py
View File

@ -126,7 +126,8 @@ class ConnectionItem(common.WWrap):
self.master.killextra, self.master.rheaders, self.master.killextra, self.master.rheaders,
False, self.master.nopop, False, self.master.nopop,
self.master.options.replay_ignore_params, self.master.options.replay_ignore_content, self.master.options.replay_ignore_params, self.master.options.replay_ignore_content,
self.master.options.replay_ignore_payload_params self.master.options.replay_ignore_payload_params,
self.master.options.replay_ignore_host
) )
elif k == "t": elif k == "t":
self.master.start_server_playback( self.master.start_server_playback(
@ -134,7 +135,8 @@ class ConnectionItem(common.WWrap):
self.master.killextra, self.master.rheaders, self.master.killextra, self.master.rheaders,
False, self.master.nopop, False, self.master.nopop,
self.master.options.replay_ignore_params, self.master.options.replay_ignore_content, self.master.options.replay_ignore_params, self.master.options.replay_ignore_content,
self.master.options.replay_ignore_payload_params self.master.options.replay_ignore_payload_params,
self.master.options.replay_ignore_host
) )
else: else:
self.master.path_prompt( self.master.path_prompt(

3
libmproxy/dump.py
View File

@ -40,6 +40,7 @@ class Options(object):
"replay_ignore_content", "replay_ignore_content",
"replay_ignore_params", "replay_ignore_params",
"replay_ignore_payload_params", "replay_ignore_payload_params",
"replay_ignore_host"
] ]
def __init__(self, **kwargs): def __init__(self, **kwargs):
@ -78,6 +79,7 @@ class DumpMaster(flow.FlowMaster):
self.showhost = options.showhost self.showhost = options.showhost
self.replay_ignore_params = options.replay_ignore_params self.replay_ignore_params = options.replay_ignore_params
self.replay_ignore_content = options.replay_ignore_content self.replay_ignore_content = options.replay_ignore_content
self.replay_ignore_host = options.replay_ignore_host
self.refresh_server_playback = options.refresh_server_playback self.refresh_server_playback = options.refresh_server_playback
self.replay_ignore_payload_params = options.replay_ignore_payload_params self.replay_ignore_payload_params = options.replay_ignore_payload_params
@ -119,6 +121,7 @@ class DumpMaster(flow.FlowMaster):
options.replay_ignore_params, options.replay_ignore_params,
options.replay_ignore_content, options.replay_ignore_content,
options.replay_ignore_payload_params, options.replay_ignore_payload_params,
options.replay_ignore_host
) )
if options.client_replay: if options.client_replay:

27
libmproxy/flow.py
View File

@ -203,12 +203,19 @@ class ClientPlaybackState:
class ServerPlaybackState: class ServerPlaybackState:
def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content, ignore_payload_params): def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content,
ignore_payload_params, ignore_host):
""" """
headers: Case-insensitive list of request headers that should be headers: Case-insensitive list of request headers that should be
included in request-response matching. included in request-response matching.
""" """
self.headers, self.exit, self.nopop, self.ignore_params, self.ignore_content, self.ignore_payload_params = headers, exit, nopop, ignore_params, ignore_content, ignore_payload_params self.headers = headers
self.exit = exit
self.nopop = nopop
self.ignore_params = ignore_params
self.ignore_content = ignore_content
self.ignore_payload_params = ignore_payload_params
self.ignore_host = ignore_host
self.fmap = {} self.fmap = {}
for i in flows: for i in flows:
if i.response: if i.response:
@ -228,7 +235,6 @@ class ServerPlaybackState:
queriesArray = urlparse.parse_qsl(query) queriesArray = urlparse.parse_qsl(query)
key = [ key = [
str(r.host),
str(r.port), str(r.port),
str(r.scheme), str(r.scheme),
str(r.method), str(r.method),
@ -245,6 +251,9 @@ class ServerPlaybackState:
else: else:
key.append(str(r.content)) key.append(str(r.content))
if not self.ignore_host:
key.append(r.host)
filtered = [] filtered = []
ignore_params = self.ignore_params or [] ignore_params = self.ignore_params or []
for p in queriesArray: for p in queriesArray:
@ -616,6 +625,7 @@ class FlowMaster(controller.Master):
self.setheaders = SetHeaders() self.setheaders = SetHeaders()
self.replay_ignore_params = False self.replay_ignore_params = False
self.replay_ignore_content = None self.replay_ignore_content = None
self.replay_ignore_host = False
self.stream = None self.stream = None
self.apps = AppRegistry() self.apps = AppRegistry()
@ -712,16 +722,19 @@ class FlowMaster(controller.Master):
def stop_client_playback(self): def stop_client_playback(self):
self.client_playback = None self.client_playback = None
def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params, ignore_content, def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params,
ignore_payload_params): ignore_content, ignore_payload_params, ignore_host):
""" """
flows: List of flows. flows: List of flows.
kill: Boolean, should we kill requests not part of the replay? kill: Boolean, should we kill requests not part of the replay?
ignore_params: list of parameters to ignore in server replay ignore_params: list of parameters to ignore in server replay
ignore_content: true if request content should be ignored in server replay ignore_content: true if request content should be ignored in server replay
ignore_payload_params: list of content params to ignore in server replay
ignore_host: true if request host should be ignored in server replay
""" """
self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, ignore_params, ignore_content, self.server_playback = ServerPlaybackState(headers, flows, exit, nopop,
ignore_payload_params) ignore_params, ignore_content,
ignore_payload_params, ignore_host)
self.kill_nonreplay = kill self.kill_nonreplay = kill
def stop_server_playback(self): def stop_server_playback(self):

42
test/test_flow.py
View File

@ -114,7 +114,7 @@ class TestClientPlaybackState:
class TestServerPlaybackState: class TestServerPlaybackState:
def test_hash(self): def test_hash(self):
s = flow.ServerPlaybackState(None, [], False, False, None, False, None) s = flow.ServerPlaybackState(None, [], False, False, None, False, None, False)
r = tutils.tflow() r = tutils.tflow()
r2 = tutils.tflow() r2 = tutils.tflow()
@ -126,7 +126,7 @@ class TestServerPlaybackState:
assert s._hash(r) != s._hash(r2) assert s._hash(r) != s._hash(r2)
def test_headers(self): def test_headers(self):
s = flow.ServerPlaybackState(["foo"], [], False, False, None, False, None) s = flow.ServerPlaybackState(["foo"], [], False, False, None, False, None, False)
r = tutils.tflow(resp=True) r = tutils.tflow(resp=True)
r.request.headers["foo"] = ["bar"] r.request.headers["foo"] = ["bar"]
r2 = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True)
@ -147,7 +147,7 @@ class TestServerPlaybackState:
r2 = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True)
r2.request.headers["key"] = ["two"] r2.request.headers["key"] = ["two"]
s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False, None) s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False, None, False)
assert s.count() == 2 assert s.count() == 2
assert len(s.fmap.keys()) == 1 assert len(s.fmap.keys()) == 1
@ -168,7 +168,7 @@ class TestServerPlaybackState:
r2 = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True)
r2.request.headers["key"] = ["two"] r2.request.headers["key"] = ["two"]
s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False, None) s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False, None, False)
assert s.count() == 2 assert s.count() == 2
s.next_flow(r) s.next_flow(r)
@ -176,7 +176,7 @@ class TestServerPlaybackState:
def test_ignore_params(self): def test_ignore_params(self):
s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False, None) s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False, None, False)
r = tutils.tflow(resp=True) r = tutils.tflow(resp=True)
r.request.path="/test?param1=1" r.request.path="/test?param1=1"
r2 = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True)
@ -190,7 +190,7 @@ class TestServerPlaybackState:
assert not s._hash(r) == s._hash(r2) assert not s._hash(r) == s._hash(r2)
def test_ignore_payload_params(self): def test_ignore_payload_params(self):
s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"]) s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"], False)
r = tutils.tflow(resp=True) r = tutils.tflow(resp=True)
r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"] r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"]
r.request.content = "paramx=x&param1=1" r.request.content = "paramx=x&param1=1"
@ -216,7 +216,7 @@ class TestServerPlaybackState:
assert not s._hash(r) == s._hash(r2) assert not s._hash(r) == s._hash(r2)
def test_ignore_payload_params_other_content_type(self): def test_ignore_payload_params_other_content_type(self):
s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"]) s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"], False)
r = tutils.tflow(resp=True) r = tutils.tflow(resp=True)
r.request.headers["Content-Type"] = ["application/json"] r.request.headers["Content-Type"] = ["application/json"]
r.request.content = '{"param1":"1"}' r.request.content = '{"param1":"1"}'
@ -231,7 +231,7 @@ class TestServerPlaybackState:
def test_ignore_payload_wins_over_params(self): def test_ignore_payload_wins_over_params(self):
#NOTE: parameters are mutually exclusive in options #NOTE: parameters are mutually exclusive in options
s = flow.ServerPlaybackState(None, [], False, False, None, True, ["param1", "param2"]) s = flow.ServerPlaybackState(None, [], False, False, None, True, ["param1", "param2"], False)
r = tutils.tflow(resp=True) r = tutils.tflow(resp=True)
r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"] r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"]
r.request.content = "paramx=y" r.request.content = "paramx=y"
@ -242,7 +242,7 @@ class TestServerPlaybackState:
assert s._hash(r) == s._hash(r2) assert s._hash(r) == s._hash(r2)
def test_ignore_content(self): def test_ignore_content(self):
s = flow.ServerPlaybackState(None, [], False, False, None, False, None) s = flow.ServerPlaybackState(None, [], False, False, None, False, None, False)
r = tutils.tflow(resp=True) r = tutils.tflow(resp=True)
r2 = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True)
@ -253,7 +253,7 @@ class TestServerPlaybackState:
assert not s._hash(r) == s._hash(r2) assert not s._hash(r) == s._hash(r2)
#now ignoring content #now ignoring content
s = flow.ServerPlaybackState(None, [], False, False, None, True, None) s = flow.ServerPlaybackState(None, [], False, False, None, True, None, False)
r = tutils.tflow(resp=True) r = tutils.tflow(resp=True)
r2 = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True)
r.request.content = "foo" r.request.content = "foo"
@ -266,6 +266,17 @@ class TestServerPlaybackState:
r2.request.content = None r2.request.content = None
assert s._hash(r) == s._hash(r2) assert s._hash(r) == s._hash(r2)
def test_ignore_host(self):
s = flow.ServerPlaybackState(None, [], False, False, None, False, None, True)
r = tutils.tflow(resp=True)
r2 = tutils.tflow(resp=True)
r.request.host="address"
r2.request.host="address"
assert s._hash(r) == s._hash(r2)
r2.request.host="wrong_address"
assert s._hash(r) == s._hash(r2)
class TestFlow: class TestFlow:
def test_copy(self): def test_copy(self):
@ -748,9 +759,8 @@ class TestFlowMaster:
f = tutils.tflow(resp=True) f = tutils.tflow(resp=True)
pb = [tutils.tflow(resp=True), f] pb = [tutils.tflow(resp=True), f]
fm = flow.FlowMaster(DummyServer(ProxyConfig()), s) fm = flow.FlowMaster(DummyServer(ProxyConfig()), s)
assert not fm.start_server_playback(pb, False, [], False, False, None, False, None) assert not fm.start_server_playback(pb, False, [], False, False, None, False, None, False)
assert not fm.start_client_playback(pb, False) assert not fm.start_client_playback(pb, False)
fm.client_playback.testing = True fm.client_playback.testing = True
@ -773,16 +783,16 @@ class TestFlowMaster:
fm.refresh_server_playback = True fm.refresh_server_playback = True
assert not fm.do_server_playback(tutils.tflow()) assert not fm.do_server_playback(tutils.tflow())
fm.start_server_playback(pb, False, [], False, False, None, False, None) fm.start_server_playback(pb, False, [], False, False, None, False, None, False)
assert fm.do_server_playback(tutils.tflow()) assert fm.do_server_playback(tutils.tflow())
fm.start_server_playback(pb, False, [], True, False, None, False, None) fm.start_server_playback(pb, False, [], True, False, None, False, None, False)
r = tutils.tflow() r = tutils.tflow()
r.request.content = "gibble" r.request.content = "gibble"
assert not fm.do_server_playback(r) assert not fm.do_server_playback(r)
assert fm.do_server_playback(tutils.tflow()) assert fm.do_server_playback(tutils.tflow())
fm.start_server_playback(pb, False, [], True, False, None, False, None) fm.start_server_playback(pb, False, [], True, False, None, False, None, False)
q = Queue.Queue() q = Queue.Queue()
fm.tick(q, 0) fm.tick(q, 0)
assert fm.should_exit.is_set() assert fm.should_exit.is_set()
@ -797,7 +807,7 @@ class TestFlowMaster:
pb = [f] pb = [f]
fm = flow.FlowMaster(None, s) fm = flow.FlowMaster(None, s)
fm.refresh_server_playback = True fm.refresh_server_playback = True
fm.start_server_playback(pb, True, [], False, False, None, False, None) fm.start_server_playback(pb, True, [], False, False, None, False, None, False)
f = tutils.tflow() f = tutils.tflow()
f.request.host = "nonexistent" f.request.host = "nonexistent"