Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2025-02-07 02:28:50 +00:00
Code Issues Projects Releases Wiki Activity

test on python3

Browse Source
This commit is contained in:
Thomas Kriechbaumer 2016-01-31 14:16:03 +01:00
parent 280b491ab2
commit e98c729bb9
6 changed files with 140 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.appveyor.yml
View File

@ -8,4 +8,4 @@ install:
- "%PYTHON%\\python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\""
build: off # Not a C# project
test_script:
- "%PYTHON%\\Scripts\\py.test -n 4"
- "%PYTHON%\\Scripts\\py.test -n 4 --timeout 10"

14
.travis.yml
View File

@ -16,8 +16,16 @@ matrix:
packages:
- libssl-dev
- python: 3.5
script:
- py.test -s --cov netlib -k "not http2"
- python: 3.5
env: OPENSSL=1.0.2
addons:
apt:
sources:
# Debian sid currently holds OpenSSL 1.0.2
# change this with future releases!
- debian-sid
packages:
- libssl-dev
- python: pypy
- python: pypy
env: OPENSSL=1.0.2
@ -58,7 +66,7 @@ before_script:
- "openssl version -a"
script:
- "py.test -s --cov netlib"
- "py.test -s --cov netlib --timeout 10"
after_success:
- coveralls

40
netlib/http/http2/connections.py
View File

@ -8,6 +8,11 @@ from .. import Headers, Response, Request
from hyperframe import frame
# TODO: remove once hyperframe released a new version > 3.1.1
# wrapper for deprecated name in old hyperframe release
frame.SettingsFrame.MAX_FRAME_SIZE = frame.SettingsFrame.SETTINGS_MAX_FRAME_SIZE
frame.SettingsFrame.MAX_HEADER_LIST_SIZE = frame.SettingsFrame.SETTINGS_MAX_HEADER_LIST_SIZE
class TCPHandler(object):
@ -35,7 +40,7 @@ class HTTP2Protocol(object):
HTTP_1_1_REQUIRED=0xd
)
CLIENT_CONNECTION_PREFACE = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
CLIENT_CONNECTION_PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n'
HTTP2_DEFAULT_SETTINGS = {
frame.SettingsFrame.HEADER_TABLE_SIZE: 4096,
@ -94,7 +99,7 @@ class HTTP2Protocol(object):
timestamp_end = time.time()
authority = headers.get(':authority', '')
authority = headers.get(':authority', b'')
method = headers.get(':method', 'GET')
scheme = headers.get(':scheme', 'https')
path = headers.get(':path', '/')
@ -113,6 +118,8 @@ class HTTP2Protocol(object):
form_in = "absolute"
# FIXME: verify if path or :host contains what we need
scheme, host, port, _ = utils.parse_url(path)
scheme = scheme.decode('ascii')
host = host.decode('ascii')
if host is None:
host = 'localhost'
@ -122,18 +129,17 @@ class HTTP2Protocol(object):
request = Request(
form_in,
method,
scheme,
host,
method.encode('ascii'),
scheme.encode('ascii'),
host.encode('ascii'),
port,
path,
(2, 0),
path.encode('ascii'),
b'2.0',
headers,
body,
timestamp_start,
timestamp_end,
)
# FIXME: We should not do this.
request.stream_id = stream_id
return request
@ -141,7 +147,7 @@ class HTTP2Protocol(object):
def read_response(
self,
__rfile,
request_method='',
request_method=b'',
body_size_limit=None,
include_body=True,
stream_id=None,
@ -170,9 +176,9 @@ class HTTP2Protocol(object):
timestamp_end = None
response = Response(
(2, 0),
b'2.0',
int(headers.get(':status', 502)),
"",
b'',
headers,
body,
timestamp_start=timestamp_start,
@ -200,13 +206,13 @@ class HTTP2Protocol(object):
headers = request.headers.copy()
if ':authority' not in headers:
headers.fields.insert(0, (':authority', bytes(authority)))
headers.fields.insert(0, (b':authority', authority.encode('ascii')))
if ':scheme' not in headers:
headers.fields.insert(0, (':scheme', bytes(request.scheme)))
headers.fields.insert(0, (b':scheme', request.scheme.encode('ascii')))
if ':path' not in headers:
headers.fields.insert(0, (':path', bytes(request.path)))
headers.fields.insert(0, (b':path', request.path.encode('ascii')))
if ':method' not in headers:
headers.fields.insert(0, (':method', bytes(request.method)))
headers.fields.insert(0, (b':method', request.method.encode('ascii')))
if hasattr(request, 'stream_id'):
stream_id = request.stream_id
@ -223,7 +229,7 @@ class HTTP2Protocol(object):
headers = response.headers.copy()
if ':status' not in headers:
headers.fields.insert(0, (':status', bytes(str(response.status_code))))
headers.fields.insert(0, (b':status', str(response.status_code).encode('ascii')))
if hasattr(response, 'stream_id'):
stream_id = response.stream_id
@ -419,7 +425,7 @@ class HTTP2Protocol(object):
self._handle_unexpected_frame(frm)
headers = Headers(
[[str(k), str(v)] for k, v in self.decoder.decode(header_blocks)]
[[k.encode('ascii'), v.encode('ascii')] for k, v in self.decoder.decode(header_blocks)]
)
return stream_id, headers, body

14
netlib/utils.py
View File

@ -369,17 +369,17 @@ def multipartdecode(headers, content):
def http2_read_raw_frame(rfile):
field = rfile.peek(3)
length = int(codecs.encode(field, 'hex_codec'), 16)
header = rfile.safe_read(9)
length = int(codecs.encode(header[:3], 'hex_codec'), 16)
if length == 4740180:
raise ValueError("Length field looks more like HTTP/1.1: %s" % rfile.peek(20))
raw = rfile.safe_read(9 + length)
return raw
body = rfile.safe_read(length)
return [header, body]
def http2_read_frame(rfile):
raw = http2_read_raw_frame(rfile)
frame, length = hyperframe.frame.Frame.parse_frame_header(raw[:9])
frame.parse_body(memoryview(raw[9:]))
header, body = http2_read_raw_frame(rfile)
frame, length = hyperframe.frame.Frame.parse_frame_header(header)
frame.parse_body(memoryview(body))
return frame

1
setup.py
View File

@ -64,6 +64,7 @@ setup(
"pytest>=2.8.0",
"pytest-xdist>=1.13.1",
"pytest-cov>=2.1.0",
"pytest-timeout>=1.0.0",
"coveralls>=0.4.1",
"autopep8>=1.0.3",
"autoflake>=0.6.6",

200
test/http/http2/test_connections.py
View File

@ -4,11 +4,12 @@ import codecs
from hyperframe.frame import *
from netlib import tcp, http, utils, tutils, tservers
from netlib import tcp, http, utils, tservers
from netlib.tutils import raises
from netlib.exceptions import TcpDisconnect
from netlib.http import Headers
from netlib.http.http2.connections import HTTP2Protocol, TCPHandler
class TestTCPHandlerWrapper:
def test_wrapped(self):
h = TCPHandler(rfile='foo', wfile='bar')
@ -92,35 +93,33 @@ class TestCheckALPNMismatch(tservers.ServerTestBase):
c.connect()
c.convert_to_ssl(alpn_protos=[b'h2'])
protocol = HTTP2Protocol(c)
tutils.raises(NotImplementedError, protocol.check_alpn)
with raises(NotImplementedError):
protocol.check_alpn()
class TestPerformServerConnectionPreface(tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
try:
# send magic
self.wfile.write(codecs.decode('505249202a20485454502f322e300d0a0d0a534d0d0a0d0a', 'hex_codec'))
self.wfile.flush()
# send magic
self.wfile.write(codecs.decode('505249202a20485454502f322e300d0a0d0a534d0d0a0d0a', 'hex_codec'))
self.wfile.flush()
# send empty settings frame
self.wfile.write(codecs.decode('000000040000000000', 'hex_codec'))
self.wfile.flush()
# send empty settings frame
self.wfile.write(codecs.decode('000000040000000000', 'hex_codec'))
self.wfile.flush()
# check empty settings frame
raw = utils.http2_read_raw_frame(self.rfile)
assert raw == codecs.decode('00000c040000000000000200000000000300000001', 'hex_codec')
# check empty settings frame
raw = utils.http2_read_raw_frame(self.rfile)
assert raw == codecs.decode('00000c040000000000000200000000000300000001', 'hex_codec')
# check settings acknowledgement
raw = utils.http2_read_raw_frame(self.rfile)
assert raw == codecs.decode('000000040100000000', 'hex_codec')
# check settings acknowledgement
raw = utils.http2_read_raw_frame(self.rfile)
assert raw == codecs.decode('000000040100000000', 'hex_codec')
# send settings acknowledgement
self.wfile.write(codecs.decode('000000040100000000', 'hex_codec'))
self.wfile.flush()
except Exception as e:
print(e)
# send settings acknowledgement
self.wfile.write(codecs.decode('000000040100000000', 'hex_codec'))
self.wfile.flush()
def test_perform_server_connection_preface(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
@ -131,11 +130,8 @@ class TestPerformServerConnectionPreface(tservers.ServerTestBase):
protocol.perform_server_connection_preface()
assert protocol.connection_preface_performed
frm = protocol.read_frame()
assert isinstance(frm, SettingsFrame)
assert 'ACK' in frm.flags
tutils.raises(TcpDisconnect, protocol.perform_server_connection_preface, force=True)
with raises(TcpDisconnect):
protocol.perform_server_connection_preface(force=True)
class TestPerformClientConnectionPreface(tservers.ServerTestBase):
@ -143,23 +139,22 @@ class TestPerformClientConnectionPreface(tservers.ServerTestBase):
def handle(self):
# check magic
assert self.rfile.read(24) ==\
'505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex')
assert self.rfile.read(24) == HTTP2Protocol.CLIENT_CONNECTION_PREFACE
# check empty settings frame
assert self.rfile.read(9) ==\
'000000040000000000'.decode('hex')
codecs.decode('000000040000000000', 'hex_codec')
# send empty settings frame
self.wfile.write('000000040000000000'.decode('hex'))
self.wfile.write(codecs.decode('000000040000000000', 'hex_codec'))
self.wfile.flush()
# check settings acknowledgement
assert self.rfile.read(9) == \
'000000040100000000'.decode('hex')
codecs.decode('000000040100000000', 'hex_codec')
# send settings acknowledgement
self.wfile.write('000000040100000000'.decode('hex'))
self.wfile.write(codecs.decode('000000040100000000', 'hex_codec'))
self.wfile.flush()
def test_perform_client_connection_preface(self):
@ -172,7 +167,7 @@ class TestPerformClientConnectionPreface(tservers.ServerTestBase):
assert protocol.connection_preface_performed
class TestClientStreamIds():
class TestClientStreamIds(object):
c = tcp.TCPClient(("127.0.0.1", 0))
protocol = HTTP2Protocol(c)
@ -186,7 +181,7 @@ class TestClientStreamIds():
assert self.protocol.current_stream_id == 5
class TestServerStreamIds():
class TestServerStreamIds(object):
c = tcp.TCPClient(("127.0.0.1", 0))
protocol = HTTP2Protocol(c, is_server=True)
@ -204,7 +199,7 @@ class TestApplySettings(tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
# check settings acknowledgement
assert self.rfile.read(9) == '000000040100000000'.decode('hex')
assert self.rfile.read(9) == codecs.decode('000000040100000000', 'hex_codec')
self.wfile.write("OK")
self.wfile.flush()
self.rfile.safe_read(9) # just to keep the connection alive a bit longer
@ -223,7 +218,7 @@ class TestApplySettings(tservers.ServerTestBase):
SettingsFrame.INITIAL_WINDOW_SIZE: 'deadbeef',
})
assert c.rfile.safe_read(2) == "OK"
assert c.rfile.safe_read(2) == b"OK"
assert protocol.http2_settings[
SettingsFrame.ENABLE_PUSH] == 'foo'
@ -233,7 +228,7 @@ class TestApplySettings(tservers.ServerTestBase):
SettingsFrame.INITIAL_WINDOW_SIZE] == 'deadbeef'
class TestCreateHeaders():
class TestCreateHeaders(object):
c = tcp.TCPClient(("127.0.0.1", 0))
def test_create_headers(self):
@ -246,14 +241,12 @@ class TestCreateHeaders():
bytes = HTTP2Protocol(self.c)._create_headers(
headers, 1, end_stream=True)
assert b''.join(bytes) ==\
'000014010500000001824488355217caf3a69a3f87408294e7838c767f'\
.decode('hex')
codecs.decode('000014010500000001824488355217caf3a69a3f87408294e7838c767f', 'hex_codec')
bytes = HTTP2Protocol(self.c)._create_headers(
headers, 1, end_stream=False)
assert b''.join(bytes) ==\
'000014010400000001824488355217caf3a69a3f87408294e7838c767f'\
.decode('hex')
codecs.decode('000014010400000001824488355217caf3a69a3f87408294e7838c767f', 'hex_codec')
def test_create_headers_multiple_frames(self):
headers = http.Headers([
@ -267,41 +260,42 @@ class TestCreateHeaders():
protocol.http2_settings[SettingsFrame.MAX_FRAME_SIZE] = 8
bytes = protocol._create_headers(headers, 1, end_stream=True)
assert len(bytes) == 3
assert bytes[0] == '000008010100000001828487408294e783'.decode('hex')
assert bytes[1] == '0000080900000000018c767f7685ee5b10'.decode('hex')
assert bytes[2] == '00000209040000000163d5'.decode('hex')
assert bytes[0] == codecs.decode('000008010100000001828487408294e783', 'hex_codec')
assert bytes[1] == codecs.decode('0000080900000000018c767f7685ee5b10', 'hex_codec')
assert bytes[2] == codecs.decode('00000209040000000163d5', 'hex_codec')
class TestCreateBody():
class TestCreateBody(object):
c = tcp.TCPClient(("127.0.0.1", 0))
def test_create_body_empty(self):
protocol = HTTP2Protocol(self.c)
bytes = protocol._create_body(b'', 1)
assert b''.join(bytes) == ''.decode('hex')
assert b''.join(bytes) == b''
def test_create_body_single_frame(self):
protocol = HTTP2Protocol(self.c)
bytes = protocol._create_body(b'foobar', 1)
assert b''.join(bytes) == '000006000100000001666f6f626172'.decode('hex')
assert b''.join(bytes) == codecs.decode('000006000100000001666f6f626172', 'hex_codec')
def test_create_body_multiple_frames(self):
protocol = HTTP2Protocol(self.c)
protocol.http2_settings[SettingsFrame.MAX_FRAME_SIZE] = 5
bytes = protocol._create_body(b'foobarmehm42', 1)
assert len(bytes) == 3
assert bytes[0] == '000005000000000001666f6f6261'.decode('hex')
assert bytes[1] == '000005000000000001726d65686d'.decode('hex')
assert bytes[2] == '0000020001000000013432'.decode('hex')
assert bytes[0] == codecs.decode('000005000000000001666f6f6261', 'hex_codec')
assert bytes[1] == codecs.decode('000005000000000001726d65686d', 'hex_codec')
assert bytes[2] == codecs.decode('0000020001000000013432', 'hex_codec')
class TestReadRequest(tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
self.wfile.write(
b'000003010400000001828487'.decode('hex'))
codecs.decode('000003010400000001828487', 'hex_codec'))
self.wfile.write(
b'000006000100000001666f6f626172'.decode('hex'))
codecs.decode('000006000100000001666f6f626172', 'hex_codec'))
self.wfile.flush()
self.rfile.safe_read(9) # just to keep the connection alive a bit longer
@ -317,7 +311,7 @@ class TestReadRequest(tservers.ServerTestBase):
req = protocol.read_request(NotImplemented)
assert req.stream_id
assert req.headers.fields == [[':method', 'GET'], [':path', '/'], [':scheme', 'https']]
assert req.headers.fields == [[b':method', b'GET'], [b':path', b'/'], [b':scheme', b'https']]
assert req.content == b'foobar'
@ -325,7 +319,7 @@ class TestReadRequestRelative(tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
self.wfile.write(
b'00000c0105000000014287d5af7e4d5a777f4481f9'.decode('hex'))
codecs.decode('00000c0105000000014287d5af7e4d5a777f4481f9', 'hex_codec'))
self.wfile.flush()
ssl = True
@ -348,7 +342,7 @@ class TestReadRequestAbsolute(tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
self.wfile.write(
b'00001901050000000182448d9d29aee30c0e492c2a1170426366871c92585422e085'.decode('hex'))
codecs.decode('00001901050000000182448d9d29aee30c0e492c2a1170426366871c92585422e085', 'hex_codec'))
self.wfile.flush()
ssl = True
@ -372,9 +366,9 @@ class TestReadRequestConnect(tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
self.wfile.write(
b'00001b0105000000014287bdab4e9c17b7ff44871c92585422e08541871c92585422e085'.decode('hex'))
codecs.decode('00001b0105000000014287bdab4e9c17b7ff44871c92585422e08541871c92585422e085', 'hex_codec'))
self.wfile.write(
b'00001d0105000000014287bdab4e9c17b7ff44882f91d35d055c87a741882f91d35d055c87a7'.decode('hex'))
codecs.decode('00001d0105000000014287bdab4e9c17b7ff44882f91d35d055c87a741882f91d35d055c87a7', 'hex_codec'))
self.wfile.flush()
ssl = True
@ -403,9 +397,9 @@ class TestReadResponse(tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
self.wfile.write(
b'00000801040000002a88628594e78c767f'.decode('hex'))
codecs.decode('00000801040000002a88628594e78c767f', 'hex_codec'))
self.wfile.write(
b'00000600010000002a666f6f626172'.decode('hex'))
codecs.decode('00000600010000002a666f6f626172', 'hex_codec'))
self.wfile.flush()
self.rfile.safe_read(9) # just to keep the connection alive a bit longer
@ -420,10 +414,10 @@ class TestReadResponse(tservers.ServerTestBase):
resp = protocol.read_response(NotImplemented, stream_id=42)
assert resp.http_version == (2, 0)
assert resp.http_version == '2.0'
assert resp.status_code == 200
assert resp.msg == ""
assert resp.headers.fields == [[':status', '200'], ['etag', 'foobar']]
assert resp.msg == ''
assert resp.headers.fields == [[b':status', b'200'], [b'etag', b'foobar']]
assert resp.content == b'foobar'
assert resp.timestamp_end
@ -432,7 +426,7 @@ class TestReadEmptyResponse(tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
self.wfile.write(
b'00000801050000002a88628594e78c767f'.decode('hex'))
codecs.decode('00000801050000002a88628594e78c767f', 'hex_codec'))
self.wfile.flush()
ssl = True
@ -447,10 +441,10 @@ class TestReadEmptyResponse(tservers.ServerTestBase):
resp = protocol.read_response(NotImplemented, stream_id=42)
assert resp.stream_id == 42
assert resp.http_version == (2, 0)
assert resp.http_version == '2.0'
assert resp.status_code == 200
assert resp.msg == b''
assert resp.headers.fields == [[':status', '200'], ['etag', 'foobar']]
assert resp.msg == ''
assert resp.headers.fields == [[b':status', b'200'], [b'etag', b'foobar']]
assert resp.content == b''
@ -459,53 +453,53 @@ class TestAssembleRequest(object):
def test_request_simple(self):
bytes = HTTP2Protocol(self.c).assemble_request(http.Request(
'',
'GET',
'https',
'',
'',
'/',
(2, 0),
b'',
b'GET',
b'https',
b'',
b'',
b'/',
b'2.0',
None,
None,
))
assert len(bytes) == 1
assert bytes[0] == '00000d0105000000018284874188089d5c0b8170dc07'.decode('hex')
assert bytes[0] == codecs.decode('00000d0105000000018284874188089d5c0b8170dc07', 'hex_codec')
def test_request_with_stream_id(self):
req = http.Request(
'',
'GET',
'https',
'',
'',
'/',
(2, 0),
b'',
b'GET',
b'https',
b'',
b'',
b'/',
b'2.0',
None,
None,
)
req.stream_id = 0x42
bytes = HTTP2Protocol(self.c).assemble_request(req)
assert len(bytes) == 1
assert bytes[0] == '00000d0105000000428284874188089d5c0b8170dc07'.decode('hex')
assert bytes[0] == codecs.decode('00000d0105000000428284874188089d5c0b8170dc07', 'hex_codec')
def test_request_with_body(self):
bytes = HTTP2Protocol(self.c).assemble_request(http.Request(
'',
'GET',
'https',
'',
'',
'/',
(2, 0),
http.Headers([('foo', 'bar')]),
'foobar',
b'',
b'GET',
b'https',
b'',
b'',
b'/',
b'2.0',
http.Headers([(b'foo', b'bar')]),
b'foobar',
))
assert len(bytes) == 2
assert bytes[0] ==\
'0000150104000000018284874188089d5c0b8170dc07408294e7838c767f'.decode('hex')
codecs.decode('0000150104000000018284874188089d5c0b8170dc07408294e7838c767f', 'hex_codec')
assert bytes[1] ==\
'000006000100000001666f6f626172'.decode('hex')
codecs.decode('000006000100000001666f6f626172', 'hex_codec')
class TestAssembleResponse(object):
@ -513,34 +507,34 @@ class TestAssembleResponse(object):
def test_simple(self):
bytes = HTTP2Protocol(self.c, is_server=True).assemble_response(http.Response(
(2, 0),
b'2.0',
200,
))
assert len(bytes) == 1
assert bytes[0] ==\
'00000101050000000288'.decode('hex')
codecs.decode('00000101050000000288', 'hex_codec')
def test_with_stream_id(self):
resp = http.Response(
(2, 0),
b'2.0',
200,
)
resp.stream_id = 0x42
bytes = HTTP2Protocol(self.c, is_server=True).assemble_response(resp)
assert len(bytes) == 1
assert bytes[0] ==\
'00000101050000004288'.decode('hex')
codecs.decode('00000101050000004288', 'hex_codec')
def test_with_body(self):
bytes = HTTP2Protocol(self.c, is_server=True).assemble_response(http.Response(
(2, 0),
b'2.0',
200,
'',
Headers(foo="bar"),
'foobar'
b'',
http.Headers(foo=b"bar"),
b'foobar'
))
assert len(bytes) == 2
assert bytes[0] ==\
'00000901040000000288408294e7838c767f'.decode('hex')
codecs.decode('00000901040000000288408294e7838c767f', 'hex_codec')
assert bytes[1] ==\
'000006000100000002666f6f626172'.decode('hex')
codecs.decode('000006000100000002666f6f626172', 'hex_codec')