diff --git a/mitmproxy/builtins/__init__.py b/mitmproxy/builtins/__init__.py index 3974d7362..5f6685707 100644 --- a/mitmproxy/builtins/__init__.py +++ b/mitmproxy/builtins/__init__.py @@ -8,6 +8,7 @@ from mitmproxy.builtins import stickycookie from mitmproxy.builtins import script from mitmproxy.builtins import replace from mitmproxy.builtins import setheaders +from mitmproxy.builtins import serverplayback def default_addons(): @@ -20,4 +21,5 @@ def default_addons(): filestreamer.FileStreamer(), replace.Replace(), setheaders.SetHeaders(), + serverplayback.ServerPlayback() ] diff --git a/mitmproxy/builtins/serverplayback.py b/mitmproxy/builtins/serverplayback.py new file mode 100644 index 000000000..fe56d68b2 --- /dev/null +++ b/mitmproxy/builtins/serverplayback.py @@ -0,0 +1,133 @@ +from __future__ import absolute_import, print_function, division +from six.moves import urllib +import hashlib + +from netlib import strutils +from mitmproxy import exceptions, flow, ctx + + +class ServerPlayback(object): + def __init__(self): + self.options = None + + self.flowmap = {} + self.stop = False + self.final_flow = None + + def load(self, flows): + for i in flows: + if i.response: + l = self.flowmap.setdefault(self._hash(i), []) + l.append(i) + + def clear(self): + self.flowmap = {} + + def count(self): + return sum([len(i) for i in self.flowmap.values()]) + + def _hash(self, flow): + """ + Calculates a loose hash of the flow request. + """ + r = flow.request + + _, _, path, _, query, _ = urllib.parse.urlparse(r.url) + queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) + + key = [str(r.port), str(r.scheme), str(r.method), str(path)] + if not self.options.replay_ignore_content: + form_contents = r.urlencoded_form or r.multipart_form + if self.options.replay_ignore_payload_params and form_contents: + params = [ + strutils.always_bytes(i) + for i in self.options.replay_ignore_payload_params + ] + for p in form_contents.items(multi=True): + if p[0] not in params: + key.append(p) + else: + key.append(str(r.raw_content)) + + if not self.options.replay_ignore_host: + key.append(r.host) + + filtered = [] + ignore_params = self.options.replay_ignore_params or [] + for p in queriesArray: + if p[0] not in ignore_params: + filtered.append(p) + for p in filtered: + key.append(p[0]) + key.append(p[1]) + + if self.options.rheaders: + headers = [] + for i in self.options.rheaders: + v = r.headers.get(i) + headers.append((i, v)) + key.append(headers) + return hashlib.sha256( + repr(key).encode("utf8", "surrogateescape") + ).digest() + + def next_flow(self, request): + """ + Returns the next flow object, or None if no matching flow was + found. + """ + hsh = self._hash(request) + if hsh in self.flowmap: + if self.options.nopop: + return self.flowmap[hsh][0] + else: + ret = self.flowmap[hsh].pop(0) + if not self.flowmap[hsh]: + del self.flowmap[hsh] + return ret + + def configure(self, options, updated): + self.options = options + if options.server_replay and "server_replay" in updated: + try: + flows = flow.read_flows_from_paths(options.server_replay) + except exceptions.FlowReadException as e: + raise exceptions.OptionsError(str(e)) + self.clear() + self.load(flows) + + # FIXME: These options have to be renamed to something more sensible - + # prefixed with serverplayback_ where appropriate, and playback_ where + # they're shared with client playback. + # + # options.kill + # options.rheaders, + # options.nopop, + # options.replay_ignore_params, + # options.replay_ignore_content, + # options.replay_ignore_payload_params, + # options.replay_ignore_host + + def tick(self): + if self.stop and not self.final_flow.live: + ctx.master.shutdown() + + def request(self, f): + if self.flowmap: + rflow = self.next_flow(f) + if rflow: + response = rflow.response.copy() + response.is_replay = True + if self.options.refresh_server_playback: + response.refresh() + f.response = response + if not self.flowmap and not self.options.keepserving: + self.final_flow = f + self.stop = True + elif self.options.kill: + ctx.log.warn( + "server_playback: killed non-replay request {}".format( + f.request.url + ) + ) + f.reply.kill() diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index 511242249..49215b3ae 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -59,18 +59,6 @@ class DumpMaster(flow.FlowMaster): "HTTP/2 is disabled. Use --no-http2 to silence this warning.", file=sys.stderr) - if options.server_replay: - self.start_server_playback( - self._readflow(options.server_replay), - options.kill, options.rheaders, - not options.keepserving, - options.nopop, - options.replay_ignore_params, - options.replay_ignore_content, - options.replay_ignore_payload_params, - options.replay_ignore_host - ) - if options.client_replay: self.start_client_playback( self._readflow(options.client_replay), diff --git a/mitmproxy/flow/__init__.py b/mitmproxy/flow/__init__.py index 8a64180e6..10e66f081 100644 --- a/mitmproxy/flow/__init__.py +++ b/mitmproxy/flow/__init__.py @@ -4,16 +4,14 @@ from mitmproxy.flow import export, modules from mitmproxy.flow.io import FlowWriter, FilteredFlowWriter, FlowReader, read_flows_from_paths from mitmproxy.flow.master import FlowMaster from mitmproxy.flow.modules import ( - AppRegistry, StreamLargeBodies, ClientPlaybackState, ServerPlaybackState + AppRegistry, StreamLargeBodies, ClientPlaybackState ) from mitmproxy.flow.state import State, FlowView -# TODO: We may want to remove the imports from .modules and just expose "modules" - __all__ = [ "export", "modules", "FlowWriter", "FilteredFlowWriter", "FlowReader", "read_flows_from_paths", "FlowMaster", "AppRegistry", "StreamLargeBodies", "ClientPlaybackState", - "ServerPlaybackState", "State", "FlowView", + "State", "FlowView", ] diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index 9cdcc8dd5..b71c2c8d3 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -29,15 +29,8 @@ class FlowMaster(controller.Master): if server: self.add_server(server) self.state = state - self.server_playback = None # type: Optional[modules.ServerPlaybackState] self.client_playback = None # type: Optional[modules.ClientPlaybackState] - self.kill_nonreplay = False - self.stream_large_bodies = None # type: Optional[modules.StreamLargeBodies] - self.replay_ignore_params = False - self.replay_ignore_content = None - self.replay_ignore_host = False - self.apps = modules.AppRegistry() def start_app(self, host, port): @@ -62,56 +55,6 @@ class FlowMaster(controller.Master): def stop_client_playback(self): self.client_playback = None - def start_server_playback( - self, - flows, - kill, - headers, - exit, - nopop, - ignore_params, - ignore_content, - ignore_payload_params, - ignore_host): - """ - flows: List of flows. - kill: Boolean, should we kill requests not part of the replay? - ignore_params: list of parameters to ignore in server replay - ignore_content: true if request content should be ignored in server replay - ignore_payload_params: list of content params to ignore in server replay - ignore_host: true if request host should be ignored in server replay - """ - self.server_playback = modules.ServerPlaybackState( - headers, - flows, - exit, - nopop, - ignore_params, - ignore_content, - ignore_payload_params, - ignore_host) - self.kill_nonreplay = kill - - def stop_server_playback(self): - self.server_playback = None - - def do_server_playback(self, flow): - """ - This method should be called by child classes in the request - handler. Returns True if playback has taken place, None if not. - """ - if self.server_playback: - rflow = self.server_playback.next_flow(flow) - if not rflow: - return None - response = rflow.response.copy() - response.is_replay = True - if self.options.refresh_server_playback: - response.refresh() - flow.response = response - return True - return None - def tick(self, timeout): if self.client_playback: stop = ( @@ -126,17 +69,6 @@ class FlowMaster(controller.Master): else: self.client_playback.tick(self) - if self.server_playback: - stop = ( - self.server_playback.count() == 0 and - self.state.active_flow_count() == 0 and - not self.kill_nonreplay - ) - exit = self.server_playback.exit - if stop: - self.stop_server_playback() - if exit: - self.shutdown() return super(FlowMaster, self).tick(timeout) def duplicate_flow(self, f): @@ -229,13 +161,6 @@ class FlowMaster(controller.Master): except IOError as v: raise exceptions.FlowReadException(v.strerror) - def process_new_request(self, f): - if self.server_playback: - pb = self.do_server_playback(f) - if not pb and self.kill_nonreplay: - self.add_log("Killed {}".format(f.request.url), "info") - f.reply.kill() - def replay_request(self, f, block=False): """ Returns None if successful, or error message if not. @@ -256,7 +181,8 @@ class FlowMaster(controller.Master): f.response = None f.error = None - self.process_new_request(f) + # FIXME: process through all addons? + # self.process_new_request(f) rt = http_replay.RequestReplayThread( self.server.config, f, @@ -314,7 +240,6 @@ class FlowMaster(controller.Master): return if f not in self.state.flows: # don't add again on replay self.state.add_flow(f) - self.process_new_request(f) return f @controller.handler diff --git a/mitmproxy/flow/modules.py b/mitmproxy/flow/modules.py index fb3c52da0..e44416c33 100644 --- a/mitmproxy/flow/modules.py +++ b/mitmproxy/flow/modules.py @@ -1,13 +1,8 @@ from __future__ import absolute_import, print_function, division -import hashlib - -from six.moves import urllib - from mitmproxy import controller from netlib import wsgi from netlib import version -from netlib import strutils from netlib.http import http1 @@ -84,95 +79,3 @@ class ClientPlaybackState: master.request(self.current) if self.current.response: master.response(self.current) - - -class ServerPlaybackState: - def __init__( - self, - headers, - flows, - exit, - nopop, - ignore_params, - ignore_content, - ignore_payload_params, - ignore_host): - """ - headers: Case-insensitive list of request headers that should be - included in request-response matching. - """ - self.headers = headers - self.exit = exit - self.nopop = nopop - self.ignore_params = ignore_params - self.ignore_content = ignore_content - self.ignore_payload_params = [strutils.always_bytes(x) for x in (ignore_payload_params or ())] - self.ignore_host = ignore_host - self.fmap = {} - for i in flows: - if i.response: - l = self.fmap.setdefault(self._hash(i), []) - l.append(i) - - def count(self): - return sum(len(i) for i in self.fmap.values()) - - def _hash(self, flow): - """ - Calculates a loose hash of the flow request. - """ - r = flow.request - - _, _, path, _, query, _ = urllib.parse.urlparse(r.url) - queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) - - key = [ - str(r.port), - str(r.scheme), - str(r.method), - str(path), - ] - - if not self.ignore_content: - form_contents = r.urlencoded_form or r.multipart_form - if self.ignore_payload_params and form_contents: - key.extend( - p for p in form_contents.items(multi=True) - if p[0] not in self.ignore_payload_params - ) - else: - key.append(str(r.raw_content)) - - if not self.ignore_host: - key.append(r.host) - - filtered = [] - ignore_params = self.ignore_params or [] - for p in queriesArray: - if p[0] not in ignore_params: - filtered.append(p) - for p in filtered: - key.append(p[0]) - key.append(p[1]) - - if self.headers: - headers = [] - for i in self.headers: - v = r.headers.get(i) - headers.append((i, v)) - key.append(headers) - return hashlib.sha256(repr(key).encode("utf8", "surrogateescape")).digest() - - def next_flow(self, request): - """ - Returns the next flow object, or None if no matching flow was - found. - """ - l = self.fmap.get(self._hash(request)) - if not l: - return None - - if self.nopop: - return l[0] - else: - return l.pop(0) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 75798381d..c49748393 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -31,6 +31,7 @@ class Options(optmanager.OptManager): anticomp=False, # type: bool client_replay=None, # type: Optional[str] kill=False, # type: bool + keepserving=True, # type: bool no_server=False, # type: bool nopop=False, # type: bool refresh_server_playback=False, # type: bool @@ -87,6 +88,7 @@ class Options(optmanager.OptManager): self.anticache = anticache self.anticomp = anticomp self.client_replay = client_replay + self.keepserving = keepserving self.kill = kill self.no_server = no_server self.nopop = nopop diff --git a/test/mitmproxy/builtins/test_serverplayback.py b/test/mitmproxy/builtins/test_serverplayback.py new file mode 100644 index 000000000..72070c7a6 --- /dev/null +++ b/test/mitmproxy/builtins/test_serverplayback.py @@ -0,0 +1,284 @@ +from .. import tutils, mastertest + +import netlib.tutils +from mitmproxy.builtins import serverplayback +from mitmproxy import options +from mitmproxy import exceptions +from mitmproxy import flow + + +class TestServerPlayback: + def test_server_playback(self): + sp = serverplayback.ServerPlayback() + sp.configure(options.Options(), []) + f = tutils.tflow(resp=True) + + assert not sp.flowmap + + sp.load([f]) + assert sp.flowmap + assert sp.next_flow(f) + assert not sp.flowmap + + def test_ignore_host(self): + sp = serverplayback.ServerPlayback() + sp.configure(options.Options(replay_ignore_host=True), []) + + r = tutils.tflow(resp=True) + r2 = tutils.tflow(resp=True) + + r.request.host = "address" + r2.request.host = "address" + assert sp._hash(r) == sp._hash(r2) + r2.request.host = "wrong_address" + assert sp._hash(r) == sp._hash(r2) + + def test_ignore_content(self): + s = serverplayback.ServerPlayback() + s.configure(options.Options(replay_ignore_content=False), []) + + r = tutils.tflow(resp=True) + r2 = tutils.tflow(resp=True) + + r.request.content = b"foo" + r2.request.content = b"foo" + assert s._hash(r) == s._hash(r2) + r2.request.content = b"bar" + assert not s._hash(r) == s._hash(r2) + + s.configure(options.Options(replay_ignore_content=True), []) + r = tutils.tflow(resp=True) + r2 = tutils.tflow(resp=True) + r.request.content = b"foo" + r2.request.content = b"foo" + assert s._hash(r) == s._hash(r2) + r2.request.content = b"bar" + assert s._hash(r) == s._hash(r2) + r2.request.content = b"" + assert s._hash(r) == s._hash(r2) + r2.request.content = None + assert s._hash(r) == s._hash(r2) + + def test_ignore_content_wins_over_params(self): + s = serverplayback.ServerPlayback() + s.configure( + options.Options( + replay_ignore_content=True, + replay_ignore_payload_params=[ + "param1", "param2" + ] + ), + [] + ) + # NOTE: parameters are mutually exclusive in options + + r = tutils.tflow(resp=True) + r.request.headers["Content-Type"] = "application/x-www-form-urlencoded" + r.request.content = b"paramx=y" + + r2 = tutils.tflow(resp=True) + r2.request.headers["Content-Type"] = "application/x-www-form-urlencoded" + r2.request.content = b"paramx=x" + + # same parameters + assert s._hash(r) == s._hash(r2) + + def test_ignore_payload_params_other_content_type(self): + s = serverplayback.ServerPlayback() + s.configure( + options.Options( + replay_ignore_content=False, + replay_ignore_payload_params=[ + "param1", "param2" + ] + ), + [] + + ) + r = tutils.tflow(resp=True) + r.request.headers["Content-Type"] = "application/json" + r.request.content = b'{"param1":"1"}' + r2 = tutils.tflow(resp=True) + r2.request.headers["Content-Type"] = "application/json" + r2.request.content = b'{"param1":"1"}' + # same content + assert s._hash(r) == s._hash(r2) + # distint content (note only x-www-form-urlencoded payload is analysed) + r2.request.content = b'{"param1":"2"}' + assert not s._hash(r) == s._hash(r2) + + def test_hash(self): + s = serverplayback.ServerPlayback() + s.configure(options.Options(), []) + + r = tutils.tflow() + r2 = tutils.tflow() + + assert s._hash(r) + assert s._hash(r) == s._hash(r2) + r.request.headers["foo"] = "bar" + assert s._hash(r) == s._hash(r2) + r.request.path = "voing" + assert s._hash(r) != s._hash(r2) + + r.request.path = "path?blank_value" + r2.request.path = "path?" + assert s._hash(r) != s._hash(r2) + + def test_headers(self): + s = serverplayback.ServerPlayback() + s.configure(options.Options(rheaders=["foo"]), []) + + r = tutils.tflow(resp=True) + r.request.headers["foo"] = "bar" + r2 = tutils.tflow(resp=True) + assert not s._hash(r) == s._hash(r2) + r2.request.headers["foo"] = "bar" + assert s._hash(r) == s._hash(r2) + r2.request.headers["oink"] = "bar" + assert s._hash(r) == s._hash(r2) + + r = tutils.tflow(resp=True) + r2 = tutils.tflow(resp=True) + assert s._hash(r) == s._hash(r2) + + def test_load(self): + s = serverplayback.ServerPlayback() + s.configure(options.Options(), []) + + r = tutils.tflow(resp=True) + r.request.headers["key"] = "one" + + r2 = tutils.tflow(resp=True) + r2.request.headers["key"] = "two" + + s.load([r, r2]) + + assert s.count() == 2 + + n = s.next_flow(r) + assert n.request.headers["key"] == "one" + assert s.count() == 1 + + n = s.next_flow(r) + assert n.request.headers["key"] == "two" + assert not s.flowmap + assert s.count() == 0 + + assert not s.next_flow(r) + + def test_load_with_nopop(self): + s = serverplayback.ServerPlayback() + s.configure(options.Options(nopop=True), []) + + r = tutils.tflow(resp=True) + r.request.headers["key"] = "one" + + r2 = tutils.tflow(resp=True) + r2.request.headers["key"] = "two" + + s.load([r, r2]) + + assert s.count() == 2 + s.next_flow(r) + assert s.count() == 2 + + def test_ignore_params(self): + s = serverplayback.ServerPlayback() + s.configure( + options.Options( + replay_ignore_params=["param1", "param2"] + ), + [] + ) + + r = tutils.tflow(resp=True) + r.request.path = "/test?param1=1" + r2 = tutils.tflow(resp=True) + r2.request.path = "/test" + assert s._hash(r) == s._hash(r2) + r2.request.path = "/test?param1=2" + assert s._hash(r) == s._hash(r2) + r2.request.path = "/test?param2=1" + assert s._hash(r) == s._hash(r2) + r2.request.path = "/test?param3=2" + assert not s._hash(r) == s._hash(r2) + + def test_ignore_payload_params(self): + s = serverplayback.ServerPlayback() + s.configure( + options.Options( + replay_ignore_payload_params=["param1", "param2"] + ), + [] + ) + + r = tutils.tflow(resp=True) + r.request.headers["Content-Type"] = "application/x-www-form-urlencoded" + r.request.content = b"paramx=x¶m1=1" + r2 = tutils.tflow(resp=True) + r2.request.headers["Content-Type"] = "application/x-www-form-urlencoded" + r2.request.content = b"paramx=x¶m1=1" + # same parameters + assert s._hash(r) == s._hash(r2) + # ignored parameters != + r2.request.content = b"paramx=x¶m1=2" + assert s._hash(r) == s._hash(r2) + # missing parameter + r2.request.content = b"paramx=x" + assert s._hash(r) == s._hash(r2) + # ignorable parameter added + r2.request.content = b"paramx=x¶m1=2" + assert s._hash(r) == s._hash(r2) + # not ignorable parameter changed + r2.request.content = b"paramx=y¶m1=1" + assert not s._hash(r) == s._hash(r2) + # not ignorable parameter missing + r2.request.content = b"param1=1" + assert not s._hash(r) == s._hash(r2) + + def test_server_playback_full(self): + state = flow.State() + s = serverplayback.ServerPlayback() + o = options.Options(refresh_server_playback = True, keepserving=False) + m = mastertest.RecordingMaster(o, None, state) + m.addons.add(o, s) + + f = tutils.tflow() + f.response = netlib.tutils.tresp(content=f.request.content) + s.load([f, f]) + + tf = tutils.tflow() + assert not tf.response + m.request(tf) + assert tf.response == f.response + + tf = tutils.tflow() + tf.request.content = b"gibble" + assert not tf.response + m.request(tf) + assert not tf.response + + assert not s.stop + s.tick() + assert not s.stop + + tf = tutils.tflow() + m.request(tutils.tflow()) + assert s.stop + + def test_server_playback_kill(self): + state = flow.State() + s = serverplayback.ServerPlayback() + o = options.Options(refresh_server_playback = True, kill=True) + m = mastertest.RecordingMaster(o, None, state) + m.addons.add(o, s) + + f = tutils.tflow() + f.response = netlib.tutils.tresp(content=f.request.content) + s.load([f]) + + f = tutils.tflow() + f.request.host = "nonexistent" + m.request(f) + assert f.reply.value == exceptions.Kill diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index 08659d193..68d88ea17 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -5,6 +5,10 @@ from mitmproxy.flow import master from mitmproxy import flow, proxy, models, controller +class TestMaster: + pass + + class MasterTest: def cycle(self, master, content): @@ -16,7 +20,9 @@ class MasterTest: master.serverconnect(f.server_conn) master.request(f) if not f.error: - f.response = models.HTTPResponse.wrap(netlib.tutils.tresp(content=content)) + f.response = models.HTTPResponse.wrap( + netlib.tutils.tresp(content=content) + ) master.response(f) master.clientdisconnect(f) return f diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index 90f33264d..40beeb0dc 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -50,7 +50,7 @@ class TestDumpMaster(mastertest.MasterTest): def test_replay(self): o = dump.Options(server_replay=["nonexistent"], kill=True) - tutils.raises(dump.DumpError, dump.DumpMaster, None, o) + tutils.raises(exceptions.OptionsError, dump.DumpMaster, None, o) with tutils.tmpdir() as t: p = os.path.join(t, "rep") diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 1caeb100e..91013efc8 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -70,228 +70,6 @@ class TestClientPlaybackState: assert not fm.client_playback -class TestServerPlaybackState: - - def test_hash(self): - s = flow.ServerPlaybackState( - None, - [], - False, - False, - None, - False, - None, - False) - r = tutils.tflow() - r2 = tutils.tflow() - - assert s._hash(r) - assert s._hash(r) == s._hash(r2) - r.request.headers["foo"] = "bar" - assert s._hash(r) == s._hash(r2) - r.request.path = "voing" - assert s._hash(r) != s._hash(r2) - - r.request.path = "path?blank_value" - r2.request.path = "path?" - assert s._hash(r) != s._hash(r2) - - def test_headers(self): - s = flow.ServerPlaybackState( - ["foo"], - [], - False, - False, - None, - False, - None, - False) - r = tutils.tflow(resp=True) - r.request.headers["foo"] = "bar" - r2 = tutils.tflow(resp=True) - assert not s._hash(r) == s._hash(r2) - r2.request.headers["foo"] = "bar" - assert s._hash(r) == s._hash(r2) - r2.request.headers["oink"] = "bar" - assert s._hash(r) == s._hash(r2) - - r = tutils.tflow(resp=True) - r2 = tutils.tflow(resp=True) - assert s._hash(r) == s._hash(r2) - - def test_load(self): - r = tutils.tflow(resp=True) - r.request.headers["key"] = "one" - - r2 = tutils.tflow(resp=True) - r2.request.headers["key"] = "two" - - s = flow.ServerPlaybackState( - None, [ - r, r2], False, False, None, False, None, False) - assert s.count() == 2 - assert len(s.fmap.keys()) == 1 - - n = s.next_flow(r) - assert n.request.headers["key"] == "one" - assert s.count() == 1 - - n = s.next_flow(r) - assert n.request.headers["key"] == "two" - assert s.count() == 0 - - assert not s.next_flow(r) - - def test_load_with_nopop(self): - r = tutils.tflow(resp=True) - r.request.headers["key"] = "one" - - r2 = tutils.tflow(resp=True) - r2.request.headers["key"] = "two" - - s = flow.ServerPlaybackState( - None, [ - r, r2], False, True, None, False, None, False) - - assert s.count() == 2 - s.next_flow(r) - assert s.count() == 2 - - def test_ignore_params(self): - s = flow.ServerPlaybackState( - None, [], False, False, [ - "param1", "param2"], False, None, False) - r = tutils.tflow(resp=True) - r.request.path = "/test?param1=1" - r2 = tutils.tflow(resp=True) - r2.request.path = "/test" - assert s._hash(r) == s._hash(r2) - r2.request.path = "/test?param1=2" - assert s._hash(r) == s._hash(r2) - r2.request.path = "/test?param2=1" - assert s._hash(r) == s._hash(r2) - r2.request.path = "/test?param3=2" - assert not s._hash(r) == s._hash(r2) - - def test_ignore_payload_params(self): - s = flow.ServerPlaybackState( - None, [], False, False, None, False, [ - "param1", "param2"], False) - r = tutils.tflow(resp=True) - r.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r.request.content = b"paramx=x¶m1=1" - r2 = tutils.tflow(resp=True) - r2.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r2.request.content = b"paramx=x¶m1=1" - # same parameters - assert s._hash(r) == s._hash(r2) - # ignored parameters != - r2.request.content = b"paramx=x¶m1=2" - assert s._hash(r) == s._hash(r2) - # missing parameter - r2.request.content = b"paramx=x" - assert s._hash(r) == s._hash(r2) - # ignorable parameter added - r2.request.content = b"paramx=x¶m1=2" - assert s._hash(r) == s._hash(r2) - # not ignorable parameter changed - r2.request.content = b"paramx=y¶m1=1" - assert not s._hash(r) == s._hash(r2) - # not ignorable parameter missing - r2.request.content = b"param1=1" - assert not s._hash(r) == s._hash(r2) - - def test_ignore_payload_params_other_content_type(self): - s = flow.ServerPlaybackState( - None, [], False, False, None, False, [ - "param1", "param2"], False) - r = tutils.tflow(resp=True) - r.request.headers["Content-Type"] = "application/json" - r.request.content = b'{"param1":"1"}' - r2 = tutils.tflow(resp=True) - r2.request.headers["Content-Type"] = "application/json" - r2.request.content = b'{"param1":"1"}' - # same content - assert s._hash(r) == s._hash(r2) - # distint content (note only x-www-form-urlencoded payload is analysed) - r2.request.content = b'{"param1":"2"}' - assert not s._hash(r) == s._hash(r2) - - def test_ignore_payload_wins_over_params(self): - # NOTE: parameters are mutually exclusive in options - s = flow.ServerPlaybackState( - None, [], False, False, None, True, [ - "param1", "param2"], False) - r = tutils.tflow(resp=True) - r.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r.request.content = b"paramx=y" - r2 = tutils.tflow(resp=True) - r2.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r2.request.content = b"paramx=x" - # same parameters - assert s._hash(r) == s._hash(r2) - - def test_ignore_content(self): - s = flow.ServerPlaybackState( - None, - [], - False, - False, - None, - False, - None, - False) - r = tutils.tflow(resp=True) - r2 = tutils.tflow(resp=True) - - r.request.content = b"foo" - r2.request.content = b"foo" - assert s._hash(r) == s._hash(r2) - r2.request.content = b"bar" - assert not s._hash(r) == s._hash(r2) - - # now ignoring content - s = flow.ServerPlaybackState( - None, - [], - False, - False, - None, - True, - None, - False) - r = tutils.tflow(resp=True) - r2 = tutils.tflow(resp=True) - r.request.content = b"foo" - r2.request.content = b"foo" - assert s._hash(r) == s._hash(r2) - r2.request.content = b"bar" - assert s._hash(r) == s._hash(r2) - r2.request.content = b"" - assert s._hash(r) == s._hash(r2) - r2.request.content = None - assert s._hash(r) == s._hash(r2) - - def test_ignore_host(self): - s = flow.ServerPlaybackState( - None, - [], - False, - False, - None, - False, - None, - True) - r = tutils.tflow(resp=True) - r2 = tutils.tflow(resp=True) - - r.request.host = "address" - r2.request.host = "address" - assert s._hash(r) == s._hash(r2) - r2.request.host = "wrong_address" - assert s._hash(r) == s._hash(r2) - - class TestHTTPFlow(object): def test_copy(self): @@ -753,16 +531,6 @@ class TestFlowMaster: DummyServer(ProxyConfig(options.Options())), s ) - assert not fm.start_server_playback( - pb, - False, - [], - False, - False, - None, - False, - None, - False) assert not fm.start_client_playback(pb, False) fm.client_playback.testing = True @@ -773,73 +541,6 @@ class TestFlowMaster: f.error = Error("error") fm.error(f) - def test_server_playback(self): - s = flow.State() - - f = tutils.tflow() - f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) - pb = [f] - - fm = flow.FlowMaster(options.Options(), None, s) - fm.refresh_server_playback = True - assert not fm.do_server_playback(tutils.tflow()) - - fm.start_server_playback( - pb, - False, - [], - False, - False, - None, - False, - None, - False) - assert fm.do_server_playback(tutils.tflow()) - - fm.start_server_playback( - pb, - False, - [], - True, - False, - None, - False, - None, - False) - r = tutils.tflow() - r.request.content = b"gibble" - assert not fm.do_server_playback(r) - assert fm.do_server_playback(tutils.tflow()) - - fm.tick(0) - assert fm.should_exit.is_set() - - fm.stop_server_playback() - assert not fm.server_playback - - def test_server_playback_kill(self): - s = flow.State() - f = tutils.tflow() - f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) - pb = [f] - fm = flow.FlowMaster(None, None, s) - fm.refresh_server_playback = True - fm.start_server_playback( - pb, - True, - [], - False, - False, - None, - False, - None, - False) - - f = tutils.tflow() - f.request.host = "nonexistent" - fm.request(f) - assert f.reply.value == Kill - class TestRequest: