Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-23 08:11:00 +00:00
Code Issues Projects Releases Wiki Activity

client replay: expad and consolidate tests

Browse Source
This commit is contained in:
Aldo Cortesi 2018-05-01 07:24:53 +12:00 committed by Aldo Cortesi
parent e8dac2d290
commit 6d27b28b85
4 changed files with 128 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
mitmproxy/addons/clientplayback.py
View File

@ -34,10 +34,10 @@ class RequestReplayThread(basethread.BaseThread):
def run(self): def run(self):
while True: while True:
f = self.queue.get(block=True, timeout=None) f = self.queue.get()
self.replay(f) self.replay(f)
def replay(self, f): def replay(self, f): # pragma: no cover
f.live = True f.live = True
r = f.request r = f.request
bsl = human.parse_size(self.options.body_size_limit) bsl = human.parse_size(self.options.body_size_limit)
@ -118,12 +118,13 @@ class RequestReplayThread(basethread.BaseThread):
f.live = False f.live = False
if server.connected(): if server.connected():
server.finish() server.finish()
server.close()
class ClientPlayback: class ClientPlayback:
def __init__(self): def __init__(self):
self.q: queue.Queue = queue.Queue() self.q = queue.Queue()
self.thread: RequestReplayThread | None = None self.thread: RequestReplayThread = None
def check(self, f: http.HTTPFlow): def check(self, f: http.HTTPFlow):
if f.live: if f.live:
@ -184,23 +185,25 @@ class ClientPlayback:
""" """
lst = [] lst = []
for f in flows: for f in flows:
err = self.check(f) hf = typing.cast(http.HTTPFlow, f)
err = self.check(hf)
if err: if err:
ctx.log.warn(err) ctx.log.warn(err)
continue continue
lst.append(f) lst.append(hf)
# Prepare the flow for replay # Prepare the flow for replay
f.backup() hf.backup()
f.request.is_replay = True hf.request.is_replay = True
f.response = None hf.response = None
f.error = None hf.error = None
# https://github.com/mitmproxy/mitmproxy/issues/2197 # https://github.com/mitmproxy/mitmproxy/issues/2197
if f.request.http_version == "HTTP/2.0": if hf.request.http_version == "HTTP/2.0":
f.request.http_version = "HTTP/1.1" hf.request.http_version = "HTTP/1.1"
host = f.request.headers.pop(":authority") host = hf.request.headers.pop(":authority")
f.request.headers.insert(0, "host", host) hf.request.headers.insert(0, "host", host)
self.q.put(f) self.q.put(hf)
ctx.master.addons.trigger("update", lst) ctx.master.addons.trigger("update", lst)
@command.command("replay.client.file") @command.command("replay.client.file")

11
mitmproxy/net/tcp.py
View File

@ -372,12 +372,11 @@ class TCPClient(_Connection):
# Make sure to close the real socket, not the SSL proxy. # Make sure to close the real socket, not the SSL proxy.
# OpenSSL is really good at screwing up, i.e. when trying to recv from a failed connection, # OpenSSL is really good at screwing up, i.e. when trying to recv from a failed connection,
# it tries to renegotiate... # it tries to renegotiate...
if not self.connection: if self.connection:
return if isinstance(self.connection, SSL.Connection):
elif isinstance(self.connection, SSL.Connection): close_socket(self.connection._socket)
close_socket(self.connection._socket) else:
else: close_socket(self.connection)
close_socket(self.connection)
def convert_to_tls(self, sni=None, alpn_protos=None, **sslctx_kwargs): def convert_to_tls(self, sni=None, alpn_protos=None, **sslctx_kwargs):
context = tls.create_client_context( context = tls.create_client_context(

180
test/mitmproxy/addons/test_clientplayback.py
View File

@ -1,13 +1,16 @@
import time
import pytest import pytest
from unittest import mock
from mitmproxy.test import tflow from mitmproxy.test import tflow, tutils
from mitmproxy import io from mitmproxy import io
from mitmproxy import exceptions from mitmproxy import exceptions
from mitmproxy.net import http as net_http
from mitmproxy.addons import clientplayback from mitmproxy.addons import clientplayback
from mitmproxy.test import taddons from mitmproxy.test import taddons
from .. import tservers
def tdump(path, flows): def tdump(path, flows):
with open(path, "wb") as f: with open(path, "wb") as f:
@ -21,77 +24,80 @@ class MockThread():
return False return False
class TBase(tservers.HTTPProxyTest):
@staticmethod
def wait_response(flow):
"""
Race condition: We don't want to replay the flow while it is still live.
"""
s = time.time()
while True:
if flow.response or flow.error:
break
time.sleep(0.001)
if time.time() - s > 5:
raise RuntimeError("Flow is live for too long.")
@staticmethod
def reset(f):
f.live = False
f.repsonse = False
f.error = False
def addons(self):
return [clientplayback.ClientPlayback()]
def test_replay(self):
cr = self.master.addons.get("clientplayback")
assert self.pathod("304").status_code == 304
assert len(self.master.state.flows) == 1
l = self.master.state.flows[-1]
assert l.response.status_code == 304
l.request.path = "/p/305"
cr.start_replay([l])
self.wait_response(l)
assert l.response.status_code == 305
# Disconnect error
cr.stop_replay()
self.reset(l)
l.request.path = "/p/305:d0"
cr.start_replay([l])
self.wait_response(l)
if isinstance(self, tservers.HTTPUpstreamProxyTest):
assert l.response.status_code == 502
else:
assert l.error
# # Port error
cr.stop_replay()
self.reset(l)
l.request.port = 1
# In upstream mode, we get a 502 response from the upstream proxy server.
# In upstream mode with ssl, the replay will fail as we cannot establish
# SSL with the upstream proxy.
cr.start_replay([l])
self.wait_response(l)
if isinstance(self, tservers.HTTPUpstreamProxyTest):
assert l.response.status_code == 502
else:
assert l.error
class TestHTTPProxy(TBase, tservers.HTTPProxyTest):
pass
class TestHTTPSProxy(TBase, tservers.HTTPProxyTest):
ssl = True
class TestUpstreamProxy(TBase, tservers.HTTPUpstreamProxyTest):
pass
class TestClientPlayback: class TestClientPlayback:
# @staticmethod
# def wait_until_not_live(flow):
# """
# Race condition: We don't want to replay the flow while it is still live.
# """
# s = time.time()
# while flow.live:
# time.sleep(0.001)
# if time.time() - s > 5:
# raise RuntimeError("Flow is live for too long.")
# def test_replay(self):
# assert self.pathod("304").status_code == 304
# assert len(self.master.state.flows) == 1
# l = self.master.state.flows[-1]
# assert l.response.status_code == 304
# l.request.path = "/p/305"
# self.wait_until_not_live(l)
# rt = self.master.replay_request(l, block=True)
# assert l.response.status_code == 305
# # Disconnect error
# l.request.path = "/p/305:d0"
# rt = self.master.replay_request(l, block=True)
# assert rt
# if isinstance(self, tservers.HTTPUpstreamProxyTest):
# assert l.response.status_code == 502
# else:
# assert l.error
# # Port error
# l.request.port = 1
# # In upstream mode, we get a 502 response from the upstream proxy server.
# # In upstream mode with ssl, the replay will fail as we cannot establish
# # SSL with the upstream proxy.
# rt = self.master.replay_request(l, block=True)
# assert rt
# if isinstance(self, tservers.HTTPUpstreamProxyTest):
# assert l.response.status_code == 502
# else:
# assert l.error
# def test_replay(self):
# opts = options.Options()
# fm = master.Master(opts)
# f = tflow.tflow(resp=True)
# f.request.content = None
# with pytest.raises(ReplayException, match="missing"):
# fm.replay_request(f)
# f.request = None
# with pytest.raises(ReplayException, match="request"):
# fm.replay_request(f)
# f.intercepted = True
# with pytest.raises(ReplayException, match="intercepted"):
# fm.replay_request(f)
# f.live = True
# with pytest.raises(ReplayException, match="live"):
# fm.replay_request(f)
# req = tutils.treq(headers=net_http.Headers(((b":authority", b"foo"), (b"header", b"qvalue"), (b"content-length", b"7"))))
# f = tflow.tflow(req=req)
# f.request.http_version = "HTTP/2.0"
# with mock.patch('mitmproxy.proxy.protocol.http_replay.RequestReplayThread.run'):
# rt = fm.replay_request(f)
# assert rt.f.request.http_version == "HTTP/1.1"
# assert ":authority" not in rt.f.request.headers
def test_load_file(self, tmpdir): def test_load_file(self, tmpdir):
cp = clientplayback.ClientPlayback() cp = clientplayback.ClientPlayback()
with taddons.context(cp): with taddons.context(cp):
@ -133,13 +139,37 @@ class TestClientPlayback:
f.request.raw_content = None f.request.raw_content = None
assert "missing content" in cp.check(f) assert "missing content" in cp.check(f)
def test_playback(self): @pytest.mark.asyncio
async def test_playback(self):
cp = clientplayback.ClientPlayback() cp = clientplayback.ClientPlayback()
with taddons.context(cp): with taddons.context(cp) as ctx:
assert cp.count() == 0 assert cp.count() == 0
f = tflow.tflow(resp=True) f = tflow.tflow(resp=True)
cp.start_replay([f]) cp.start_replay([f])
assert cp.count() == 1 assert cp.count() == 1
cp.stop_replay() cp.stop_replay()
assert cp.count() == 0 assert cp.count() == 0
f.live = True
cp.start_replay([f])
assert cp.count() == 0
await ctx.master.await_log("live")
def test_http2(self):
cp = clientplayback.ClientPlayback()
with taddons.context(cp):
req = tutils.treq(
headers = net_http.Headers(
(
(b":authority", b"foo"),
(b"header", b"qvalue"),
(b"content-length", b"7")
)
)
)
f = tflow.tflow(req=req)
f.request.http_version = "HTTP/2.0"
cp.start_replay([f])
assert f.request.http_version == "HTTP/1.1"
assert ":authority" not in f.request.headers

1
test/mitmproxy/tools/web/test_app.py
View File

@ -9,7 +9,6 @@ import tornado.testing
from tornado import httpclient from tornado import httpclient
from tornado import websocket from tornado import websocket
from mitmproxy import exceptions
from mitmproxy import options from mitmproxy import options
from mitmproxy.test import tflow from mitmproxy.test import tflow
from mitmproxy.tools.web import app from mitmproxy.tools.web import app