Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-26 18:18:25 +00:00
Code Issues Projects Releases Wiki Activity

Roll out synchronisation for mitmproxy tests

Browse Source 
This extends some of the work I did for pathod and netlib to the mitmproxy test
suite. It also fixes what may be a leak in replays.

Failing on connection leak is disabled on Windows for the moment.

Fixes #1535
This commit is contained in:
Aldo Cortesi 2016-09-10 09:18:11 +12:00
parent ea49b8a2e2
commit 4ff8a72521
11 changed files with 254 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
mitmproxy/addons.py
View File

@ -4,7 +4,7 @@ import pprint
def _get_name(itm):
return getattr(itm, "name", itm.__class__.__name__)
return getattr(itm, "name", itm.__class__.__name__.lower())
class Addons(object):
@ -13,6 +13,16 @@ class Addons(object):
self.master = master
master.options.changed.connect(self.options_update)
def get(self, name):
"""
Retrieve an addon by name. Addon names are equal to the .name
attribute on the instance, or the lower case class name if that
does not exist.
"""
for i in self.chain:
if name == _get_name(i):
return i
def options_update(self, options, updated):
for i in self.chain:
with self.master.handlecontext():
@ -39,14 +49,6 @@ class Addons(object):
for i in self.chain:
self.invoke_with_context(i, "done")
def has_addon(self, name):
"""
Is an addon with this name registered?
"""
for i in self.chain:
if _get_name(i) == name:
return True
def __len__(self):
return len(self.chain)

5
mitmproxy/builtins/serverplayback.py
View File

@ -88,12 +88,13 @@ class ServerPlayback(object):
def configure(self, options, updated):
self.options = options
if options.server_replay and "server_replay" in updated:
if "server_replay" in updated:
self.clear()
if options.server_replay:
try:
flows = flow.read_flows_from_paths(options.server_replay)
except exceptions.FlowReadException as e:
raise exceptions.OptionsError(str(e))
self.clear()
self.load(flows)
# FIXME: These options have to be renamed to something more sensible -

18
mitmproxy/console/master.py
View File

@ -248,9 +248,6 @@ class ConsoleMaster(flow.FlowMaster):
if options.client_replay:
self.client_playback_path(options.client_replay)
if options.server_replay:
self.server_playback_path(options.server_replay)
self.view_stack = []
if options.app:
@ -391,21 +388,6 @@ class ConsoleMaster(flow.FlowMaster):
if flows:
self.start_client_playback(flows, False)
def server_playback_path(self, path):
if not isinstance(path, list):
path = [path]
flows = self._readflows(path)
if flows:
self.start_server_playback(
flows,
self.options.kill, self.options.rheaders,
False, self.options.nopop,
self.options.replay_ignore_params,
self.options.replay_ignore_content,
self.options.replay_ignore_payload_params,
self.options.replay_ignore_host
)
def spawn_editor(self, data):
text = not isinstance(data, bytes)
fd, name = tempfile.mkstemp('', "mproxy", text=text)

10
mitmproxy/console/statusbar.py
View File

@ -147,14 +147,12 @@ class StatusBar(urwid.WidgetWrap):
if self.master.client_playback:
r.append("[")
r.append(("heading_key", "cplayback"))
r.append(":%s to go]" % self.master.client_playback.count())
if self.master.server_playback:
r.append(":%s]" % self.master.client_playback.count())
if self.master.options.server_replay:
r.append("[")
r.append(("heading_key", "splayback"))
if self.master.options.nopop:
r.append(":%s in file]" % self.master.server_playback.count())
else:
r.append(":%s to go]" % self.master.server_playback.count())
a = self.master.addons.get("serverplayback")
r.append(":%s]" % a.count())
if self.master.options.ignore_hosts:
r.append("[")
r.append(("heading_key", "I"))

20
mitmproxy/console/window.py
View File

@ -57,13 +57,11 @@ class Window(urwid.Frame):
callback = self.master.stop_client_playback_prompt,
)
elif k == "s":
if not self.master.server_playback:
signals.status_prompt_path.send(
self,
prompt = "Server replay path",
callback = self.master.server_playback_path
)
else:
a = self.master.addons.get("serverplayback")
if a.count():
def stop_server_playback(response):
if response == "y":
self.master.options.server_replay = []
signals.status_prompt_onekey.send(
self,
prompt = "Stop current server replay?",
@ -71,7 +69,13 @@ class Window(urwid.Frame):
("yes", "y"),
("no", "n"),
),
callback = self.master.stop_server_playback_prompt,
callback = stop_server_playback
)
else:
signals.status_prompt_path.send(
self,
prompt = "Server playback path",
callback = lambda x: self.master.options.setter("server_replay")([x])
)
def keypress(self, size, k):

3
mitmproxy/protocol/http_replay.py
View File

@ -33,6 +33,7 @@ class RequestReplayThread(basethread.BaseThread):
def run(self):
r = self.flow.request
first_line_format_backup = r.first_line_format
server = None
try:
self.flow.response = None
@ -103,3 +104,5 @@ class RequestReplayThread(basethread.BaseThread):
self.channel.tell("log", Log(traceback.format_exc(), "error"))
finally:
r.first_line_format = first_line_format_backup
if server:
server.finish()

8
test/mitmproxy/protocol/test_http1.py
View File

@ -18,13 +18,14 @@ class TestInvalidRequests(tservers.HTTPProxyTest):
def test_double_connect(self):
p = self.pathoc()
with p.connect():
r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port))
assert r.status_code == 400
assert b"Invalid HTTP request form" in r.content
def test_relative_request(self):
p = self.pathoc_raw()
p.connect()
with p.connect():
r = p.request("get:/p/200")
assert r.status_code == 400
assert b"Invalid HTTP request form" in r.content
@ -61,5 +62,8 @@ class TestHeadContentLength(tservers.HTTPProxyTest):
def test_head_content_length(self):
p = self.pathoc()
resp = p.request("""head:'%s/p/200:h"Content-Length"="42"'""" % self.server.urlbase)
with p.connect():
resp = p.request(
"""head:'%s/p/200:h"Content-Length"="42"'""" % self.server.urlbase
)
assert resp.headers["Content-Length"] == "42"

4
test/mitmproxy/test_addons.py
View File

@ -17,5 +17,5 @@ def test_simple():
m = controller.Master(o)
a = addons.Addons(m)
a.add(o, TAddon("one"))
assert a.has_addon("one")
assert not a.has_addon("two")
assert a.get("one")
assert not a.get("two")

3
test/mitmproxy/test_fuzzing.py
View File

@ -11,16 +11,19 @@ class TestFuzzy(tservers.HTTPProxyTest):
def test_idna_err(self):
req = r'get:"http://localhost:%s":i10,"\xc6"'
p = self.pathoc()
with p.connect():
assert p.request(req % self.server.port).status_code == 400
def test_nullbytes(self):
req = r'get:"http://localhost:%s":i19,"\x00"'
p = self.pathoc()
with p.connect():
assert p.request(req % self.server.port).status_code == 400
def test_invalid_ipv6_url(self):
req = 'get:"http://localhost:%s":i13,"["'
p = self.pathoc()
with p.connect():
resp = p.request(req % self.server.port)
assert resp.status_code == 400

35
test/mitmproxy/test_server.py
View File

@ -91,7 +91,7 @@ class CommonMixin:
def test_invalid_http(self):
t = tcp.TCPClient(("127.0.0.1", self.proxy.port))
t.connect()
with t.connect():
t.wfile.write(b"invalid\r\n\r\n")
t.wfile.flush()
line = t.rfile.readline()
@ -208,19 +208,21 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin):
def test_app_err(self):
p = self.pathoc()
with p.connect():
ret = p.request("get:'http://errapp/'")
assert ret.status_code == 500
assert b"ValueError" in ret.content
def test_invalid_connect(self):
t = tcp.TCPClient(("127.0.0.1", self.proxy.port))
t.connect()
with t.connect():
t.wfile.write(b"CONNECT invalid\n\n")
t.wfile.flush()
assert b"Bad Request" in t.rfile.readline()
def test_upstream_ssl_error(self):
p = self.pathoc()
with p.connect():
ret = p.request("get:'https://localhost:%s/'" % self.server.port)
assert ret.status_code == 400
@ -232,11 +234,13 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin):
# Lets sanity check that the connection does indeed stay open by
# issuing two requests over the same connection
p = self.pathoc()
with p.connect():
assert p.request("get:'%s'" % response)
assert p.request("get:'%s'" % response)
# Now check that the connection is closed as the client specifies
p = self.pathoc()
with p.connect():
assert p.request("get:'%s':h'Connection'='close'" % response)
# There's a race here, which means we can get any of a number of errors.
# Rather than introduce yet another sleep into the test suite, we just
@ -247,6 +251,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin):
def test_reconnect(self):
req = "get:'%s/p/200:b@1:da'" % self.server.urlbase
p = self.pathoc()
with p.connect():
assert p.request(req)
# Server has disconnected. Mitmproxy should detect this, and reconnect.
assert p.request(req)
@ -260,17 +265,20 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin):
req = "get:'%s/p/200:b@1'"
p = self.pathoc()
with p.connect():
assert p.request(req % self.server.urlbase)
assert p.request(req % self.server2.urlbase)
assert switched(self.proxy.tlog)
def test_blank_leading_line(self):
p = self.pathoc()
with p.connect():
req = "get:'%s/p/201':i0,'\r\n'"
assert p.request(req % self.server.urlbase).status_code == 201
def test_invalid_headers(self):
p = self.pathoc()
with p.connect():
resp = p.request("get:'http://foo':h':foo'='bar'")
assert resp.status_code == 400
@ -301,6 +309,7 @@ class TestHTTPAuth(tservers.HTTPProxyTest):
self.master.options.auth_singleuser = "test:test"
assert self.pathod("202").status_code == 407
p = self.pathoc()
with p.connect():
ret = p.request("""
get
'http://localhost:%s/p/202'
@ -318,6 +327,7 @@ class TestHTTPReverseAuth(tservers.ReverseProxyTest):
self.master.options.auth_singleuser = "test:test"
assert self.pathod("202").status_code == 401
p = self.pathoc()
with p.connect():
ret = p.request("""
get
'/p/202'
@ -354,6 +364,7 @@ class TestHTTPS(tservers.HTTPProxyTest, CommonMixin, TcpMixin):
def test_error_post_connect(self):
p = self.pathoc()
with p.connect():
assert p.request("get:/:i0,'invalid\r\n\r\n'").status_code == 400
@ -389,6 +400,7 @@ class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest):
def _request(self):
p = self.pathoc(sni="example.mitmproxy.org")
with p.connect():
return p.request("get:/p/242")
def test_verification_w_cadir(self):
@ -426,6 +438,7 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest):
def _request(self):
p = self.pathoc(sni="example.mitmproxy.org")
with p.connect():
return p.request("get:/p/242")
@classmethod
@ -481,12 +494,14 @@ class TestSocks5(tservers.SocksModeTest):
def test_simple(self):
p = self.pathoc()
with p.connect():
p.socks_connect(("localhost", self.server.port))
f = p.request("get:/p/200")
assert f.status_code == 200
def test_with_authentication_only(self):
p = self.pathoc()
with p.connect():
f = p.request("get:/p/200")
assert f.status_code == 502
assert b"SOCKS5 mode failure" in f.content
@ -496,7 +511,7 @@ class TestSocks5(tservers.SocksModeTest):
mitmproxy doesn't support UDP or BIND SOCKS CMDs
"""
p = self.pathoc()
with p.connect():
socks.ClientGreeting(
socks.VERSION.SOCKS5,
[socks.METHOD.NO_AUTHENTICATION_REQUIRED]
@ -531,20 +546,22 @@ class TestHttps2Http(tservers.ReverseProxyTest):
p = pathoc.Pathoc(
("localhost", self.proxy.port), ssl=True, sni=sni, fp=None
)
p.connect()
return p
def test_all(self):
p = self.pathoc(ssl=True)
with p.connect():
assert p.request("get:'/p/200'").status_code == 200
def test_sni(self):
p = self.pathoc(ssl=True, sni="example.com")
with p.connect():
assert p.request("get:'/p/200'").status_code == 200
assert all("Error in handle_sni" not in msg for msg in self.proxy.tlog)
def test_http(self):
p = self.pathoc(ssl=False)
with p.connect():
assert p.request("get:'/p/200'").status_code == 200
@ -703,7 +720,7 @@ class TestRedirectRequest(tservers.HTTPProxyTest):
self.master.redirect_port = self.server2.port
p = self.pathoc()
with p.connect():
self.server.clear_log()
self.server2.clear_log()
r1 = p.request("get:'/p/200'")
@ -743,7 +760,7 @@ class TestStreamRequest(tservers.HTTPProxyTest):
def test_stream_simple(self):
p = self.pathoc()
with p.connect():
# a request with 100k of data but without content-length
r1 = p.request("get:'%s/p/200:r:b@100k:d102400'" % self.server.urlbase)
assert r1.status_code == 200
@ -751,7 +768,7 @@ class TestStreamRequest(tservers.HTTPProxyTest):
def test_stream_multiple(self):
p = self.pathoc()
with p.connect():
# simple request with streaming turned on
r1 = p.request("get:'%s/p/200'" % self.server.urlbase)
assert r1.status_code == 200
@ -887,6 +904,7 @@ class TestUpstreamProxy(tservers.HTTPUpstreamProxyTest, CommonMixin, AppMixin):
("~s", "baz", "ORLY")
]
p = self.pathoc()
with p.connect():
req = p.request("get:'%s/p/418:b\"foo\"'" % self.server.urlbase)
assert req.content == b"ORLY"
assert req.status_code == 418
@ -948,6 +966,7 @@ class TestUpstreamProxySSL(
def test_simple(self):
p = self.pathoc()
with p.connect():
req = p.request("get:'/p/418:b\"content\"'")
assert req.content == b"content"
assert req.status_code == 418
@ -1006,6 +1025,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest):
])
p = self.pathoc()
with p.connect():
req = p.request("get:'/p/418:b\"content\"'")
assert req.content == b"content"
assert req.status_code == 418
@ -1066,6 +1086,7 @@ class AddUpstreamCertsToClientChainMixin:
d = f.read()
upstreamCert = SSLCert.from_pem(d)
p = self.pathoc()
with p.connect():
upstream_cert_found_in_client_chain = False
for receivedCert in p.server_certs:
if receivedCert.digest('sha256') == upstreamCert.digest('sha256'):

39
test/mitmproxy/tservers.py
View File

@ -3,6 +3,7 @@ import threading
import tempfile
import flask
import mock
import sys
from mitmproxy.proxy.config import ProxyConfig
from mitmproxy.proxy.server import ProxyServer
@ -10,6 +11,7 @@ import pathod.test
import pathod.pathoc
from mitmproxy import flow, controller, options
from mitmproxy import builtins
import netlib.exceptions
testapp = flask.Flask(__name__)
@ -104,6 +106,14 @@ class ProxyTestBase(object):
cls.server.shutdown()
cls.server2.shutdown()
def teardown(self):
try:
self.server.wait_for_silence()
except netlib.exceptions.Timeout:
# FIXME: Track down the Windows sync issues
if sys.platform != "win32":
raise
def setup(self):
self.master.clear_log()
self.master.state.clear()
@ -125,6 +135,15 @@ class ProxyTestBase(object):
)
class LazyPathoc(pathod.pathoc.Pathoc):
def __init__(self, lazy_connect, *args, **kwargs):
self.lazy_connect = lazy_connect
pathod.pathoc.Pathoc.__init__(self, *args, **kwargs)
def connect(self):
return pathod.pathoc.Pathoc.connect(self, self.lazy_connect)
class HTTPProxyTest(ProxyTestBase):
def pathoc_raw(self):
@ -134,14 +153,14 @@ class HTTPProxyTest(ProxyTestBase):
"""
Returns a connected Pathoc instance.
"""
p = pathod.pathoc.Pathoc(
if self.ssl:
conn = ("127.0.0.1", self.server.port)
else:
conn = None
return LazyPathoc(
conn,
("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None
)
if self.ssl:
p.connect(("127.0.0.1", self.server.port))
else:
p.connect()
return p
def pathod(self, spec, sni=None):
"""
@ -152,6 +171,7 @@ class HTTPProxyTest(ProxyTestBase):
q = "get:'/p/%s'" % spec
else:
q = "get:'%s/p/%s'" % (self.server.urlbase, spec)
with p.connect():
return p.request(q)
def app(self, page):
@ -159,10 +179,11 @@ class HTTPProxyTest(ProxyTestBase):
p = pathod.pathoc.Pathoc(
("127.0.0.1", self.proxy.port), True, fp=None
)
p.connect((options.APP_HOST, options.APP_PORT))
with p.connect((options.APP_HOST, options.APP_PORT)):
return p.request("get:'%s'" % page)
else:
p = self.pathoc()
with p.connect():
return p.request("get:'http://%s%s'" % (options.APP_HOST, page))
@ -210,6 +231,7 @@ class TransparentProxyTest(ProxyTestBase):
else:
p = self.pathoc()
q = "get:'/p/%s'" % spec
with p.connect():
return p.request(q)
def pathoc(self, sni=None):
@ -219,7 +241,6 @@ class TransparentProxyTest(ProxyTestBase):
p = pathod.pathoc.Pathoc(
("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None
)
p.connect()
return p
@ -247,7 +268,6 @@ class ReverseProxyTest(ProxyTestBase):
p = pathod.pathoc.Pathoc(
("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None
)
p.connect()
return p
def pathod(self, spec, sni=None):
@ -260,6 +280,7 @@ class ReverseProxyTest(ProxyTestBase):
else:
p = self.pathoc()
q = "get:'/p/%s'" % spec
with p.connect():
return p.request(q)