From e98c729bb9b0d3debde6f61c948108bdc9dbafbe Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 31 Jan 2016 14:16:03 +0100 Subject: [PATCH] test on python3 --- .appveyor.yml | 2 +- .travis.yml | 14 +- netlib/http/http2/connections.py | 40 +++--- netlib/utils.py | 14 +- setup.py | 1 + test/http/http2/test_connections.py | 200 ++++++++++++++-------------- 6 files changed, 140 insertions(+), 131 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index dbb6d2fa6..cd1354c20 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -8,4 +8,4 @@ install: - "%PYTHON%\\python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\"" build: off # Not a C# project test_script: - - "%PYTHON%\\Scripts\\py.test -n 4" \ No newline at end of file + - "%PYTHON%\\Scripts\\py.test -n 4 --timeout 10" diff --git a/.travis.yml b/.travis.yml index 84f32d718..529c7ed36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,16 @@ matrix: packages: - libssl-dev - python: 3.5 - script: - - py.test -s --cov netlib -k "not http2" + - python: 3.5 + env: OPENSSL=1.0.2 + addons: + apt: + sources: + # Debian sid currently holds OpenSSL 1.0.2 + # change this with future releases! + - debian-sid + packages: + - libssl-dev - python: pypy - python: pypy env: OPENSSL=1.0.2 @@ -58,7 +66,7 @@ before_script: - "openssl version -a" script: - - "py.test -s --cov netlib" + - "py.test -s --cov netlib --timeout 10" after_success: - coveralls diff --git a/netlib/http/http2/connections.py b/netlib/http/http2/connections.py index c963f7c41..91133121c 100644 --- a/netlib/http/http2/connections.py +++ b/netlib/http/http2/connections.py @@ -8,6 +8,11 @@ from .. import Headers, Response, Request from hyperframe import frame +# TODO: remove once hyperframe released a new version > 3.1.1 +# wrapper for deprecated name in old hyperframe release +frame.SettingsFrame.MAX_FRAME_SIZE = frame.SettingsFrame.SETTINGS_MAX_FRAME_SIZE +frame.SettingsFrame.MAX_HEADER_LIST_SIZE = frame.SettingsFrame.SETTINGS_MAX_HEADER_LIST_SIZE + class TCPHandler(object): @@ -35,7 +40,7 @@ class HTTP2Protocol(object): HTTP_1_1_REQUIRED=0xd ) - CLIENT_CONNECTION_PREFACE = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + CLIENT_CONNECTION_PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' HTTP2_DEFAULT_SETTINGS = { frame.SettingsFrame.HEADER_TABLE_SIZE: 4096, @@ -94,7 +99,7 @@ class HTTP2Protocol(object): timestamp_end = time.time() - authority = headers.get(':authority', '') + authority = headers.get(':authority', b'') method = headers.get(':method', 'GET') scheme = headers.get(':scheme', 'https') path = headers.get(':path', '/') @@ -113,6 +118,8 @@ class HTTP2Protocol(object): form_in = "absolute" # FIXME: verify if path or :host contains what we need scheme, host, port, _ = utils.parse_url(path) + scheme = scheme.decode('ascii') + host = host.decode('ascii') if host is None: host = 'localhost' @@ -122,18 +129,17 @@ class HTTP2Protocol(object): request = Request( form_in, - method, - scheme, - host, + method.encode('ascii'), + scheme.encode('ascii'), + host.encode('ascii'), port, - path, - (2, 0), + path.encode('ascii'), + b'2.0', headers, body, timestamp_start, timestamp_end, ) - # FIXME: We should not do this. request.stream_id = stream_id return request @@ -141,7 +147,7 @@ class HTTP2Protocol(object): def read_response( self, __rfile, - request_method='', + request_method=b'', body_size_limit=None, include_body=True, stream_id=None, @@ -170,9 +176,9 @@ class HTTP2Protocol(object): timestamp_end = None response = Response( - (2, 0), + b'2.0', int(headers.get(':status', 502)), - "", + b'', headers, body, timestamp_start=timestamp_start, @@ -200,13 +206,13 @@ class HTTP2Protocol(object): headers = request.headers.copy() if ':authority' not in headers: - headers.fields.insert(0, (':authority', bytes(authority))) + headers.fields.insert(0, (b':authority', authority.encode('ascii'))) if ':scheme' not in headers: - headers.fields.insert(0, (':scheme', bytes(request.scheme))) + headers.fields.insert(0, (b':scheme', request.scheme.encode('ascii'))) if ':path' not in headers: - headers.fields.insert(0, (':path', bytes(request.path))) + headers.fields.insert(0, (b':path', request.path.encode('ascii'))) if ':method' not in headers: - headers.fields.insert(0, (':method', bytes(request.method))) + headers.fields.insert(0, (b':method', request.method.encode('ascii'))) if hasattr(request, 'stream_id'): stream_id = request.stream_id @@ -223,7 +229,7 @@ class HTTP2Protocol(object): headers = response.headers.copy() if ':status' not in headers: - headers.fields.insert(0, (':status', bytes(str(response.status_code)))) + headers.fields.insert(0, (b':status', str(response.status_code).encode('ascii'))) if hasattr(response, 'stream_id'): stream_id = response.stream_id @@ -419,7 +425,7 @@ class HTTP2Protocol(object): self._handle_unexpected_frame(frm) headers = Headers( - [[str(k), str(v)] for k, v in self.decoder.decode(header_blocks)] + [[k.encode('ascii'), v.encode('ascii')] for k, v in self.decoder.decode(header_blocks)] ) return stream_id, headers, body diff --git a/netlib/utils.py b/netlib/utils.py index c537754a9..1c1b617ac 100644 --- a/netlib/utils.py +++ b/netlib/utils.py @@ -369,17 +369,17 @@ def multipartdecode(headers, content): def http2_read_raw_frame(rfile): - field = rfile.peek(3) - length = int(codecs.encode(field, 'hex_codec'), 16) + header = rfile.safe_read(9) + length = int(codecs.encode(header[:3], 'hex_codec'), 16) if length == 4740180: raise ValueError("Length field looks more like HTTP/1.1: %s" % rfile.peek(20)) - raw = rfile.safe_read(9 + length) - return raw + body = rfile.safe_read(length) + return [header, body] def http2_read_frame(rfile): - raw = http2_read_raw_frame(rfile) - frame, length = hyperframe.frame.Frame.parse_frame_header(raw[:9]) - frame.parse_body(memoryview(raw[9:])) + header, body = http2_read_raw_frame(rfile) + frame, length = hyperframe.frame.Frame.parse_frame_header(header) + frame.parse_body(memoryview(body)) return frame diff --git a/setup.py b/setup.py index ba723bfa0..e842fd74e 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,7 @@ setup( "pytest>=2.8.0", "pytest-xdist>=1.13.1", "pytest-cov>=2.1.0", + "pytest-timeout>=1.0.0", "coveralls>=0.4.1", "autopep8>=1.0.3", "autoflake>=0.6.6", diff --git a/test/http/http2/test_connections.py b/test/http/http2/test_connections.py index edbd4f8bb..22a432660 100644 --- a/test/http/http2/test_connections.py +++ b/test/http/http2/test_connections.py @@ -4,11 +4,12 @@ import codecs from hyperframe.frame import * -from netlib import tcp, http, utils, tutils, tservers +from netlib import tcp, http, utils, tservers +from netlib.tutils import raises from netlib.exceptions import TcpDisconnect -from netlib.http import Headers from netlib.http.http2.connections import HTTP2Protocol, TCPHandler + class TestTCPHandlerWrapper: def test_wrapped(self): h = TCPHandler(rfile='foo', wfile='bar') @@ -92,35 +93,33 @@ class TestCheckALPNMismatch(tservers.ServerTestBase): c.connect() c.convert_to_ssl(alpn_protos=[b'h2']) protocol = HTTP2Protocol(c) - tutils.raises(NotImplementedError, protocol.check_alpn) + with raises(NotImplementedError): + protocol.check_alpn() class TestPerformServerConnectionPreface(tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): - try: - # send magic - self.wfile.write(codecs.decode('505249202a20485454502f322e300d0a0d0a534d0d0a0d0a', 'hex_codec')) - self.wfile.flush() + # send magic + self.wfile.write(codecs.decode('505249202a20485454502f322e300d0a0d0a534d0d0a0d0a', 'hex_codec')) + self.wfile.flush() - # send empty settings frame - self.wfile.write(codecs.decode('000000040000000000', 'hex_codec')) - self.wfile.flush() + # send empty settings frame + self.wfile.write(codecs.decode('000000040000000000', 'hex_codec')) + self.wfile.flush() - # check empty settings frame - raw = utils.http2_read_raw_frame(self.rfile) - assert raw == codecs.decode('00000c040000000000000200000000000300000001', 'hex_codec') + # check empty settings frame + raw = utils.http2_read_raw_frame(self.rfile) + assert raw == codecs.decode('00000c040000000000000200000000000300000001', 'hex_codec') - # check settings acknowledgement - raw = utils.http2_read_raw_frame(self.rfile) - assert raw == codecs.decode('000000040100000000', 'hex_codec') + # check settings acknowledgement + raw = utils.http2_read_raw_frame(self.rfile) + assert raw == codecs.decode('000000040100000000', 'hex_codec') - # send settings acknowledgement - self.wfile.write(codecs.decode('000000040100000000', 'hex_codec')) - self.wfile.flush() - except Exception as e: - print(e) + # send settings acknowledgement + self.wfile.write(codecs.decode('000000040100000000', 'hex_codec')) + self.wfile.flush() def test_perform_server_connection_preface(self): c = tcp.TCPClient(("127.0.0.1", self.port)) @@ -131,11 +130,8 @@ class TestPerformServerConnectionPreface(tservers.ServerTestBase): protocol.perform_server_connection_preface() assert protocol.connection_preface_performed - frm = protocol.read_frame() - assert isinstance(frm, SettingsFrame) - assert 'ACK' in frm.flags - - tutils.raises(TcpDisconnect, protocol.perform_server_connection_preface, force=True) + with raises(TcpDisconnect): + protocol.perform_server_connection_preface(force=True) class TestPerformClientConnectionPreface(tservers.ServerTestBase): @@ -143,23 +139,22 @@ class TestPerformClientConnectionPreface(tservers.ServerTestBase): def handle(self): # check magic - assert self.rfile.read(24) ==\ - '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex') + assert self.rfile.read(24) == HTTP2Protocol.CLIENT_CONNECTION_PREFACE # check empty settings frame assert self.rfile.read(9) ==\ - '000000040000000000'.decode('hex') + codecs.decode('000000040000000000', 'hex_codec') # send empty settings frame - self.wfile.write('000000040000000000'.decode('hex')) + self.wfile.write(codecs.decode('000000040000000000', 'hex_codec')) self.wfile.flush() # check settings acknowledgement assert self.rfile.read(9) == \ - '000000040100000000'.decode('hex') + codecs.decode('000000040100000000', 'hex_codec') # send settings acknowledgement - self.wfile.write('000000040100000000'.decode('hex')) + self.wfile.write(codecs.decode('000000040100000000', 'hex_codec')) self.wfile.flush() def test_perform_client_connection_preface(self): @@ -172,7 +167,7 @@ class TestPerformClientConnectionPreface(tservers.ServerTestBase): assert protocol.connection_preface_performed -class TestClientStreamIds(): +class TestClientStreamIds(object): c = tcp.TCPClient(("127.0.0.1", 0)) protocol = HTTP2Protocol(c) @@ -186,7 +181,7 @@ class TestClientStreamIds(): assert self.protocol.current_stream_id == 5 -class TestServerStreamIds(): +class TestServerStreamIds(object): c = tcp.TCPClient(("127.0.0.1", 0)) protocol = HTTP2Protocol(c, is_server=True) @@ -204,7 +199,7 @@ class TestApplySettings(tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): # check settings acknowledgement - assert self.rfile.read(9) == '000000040100000000'.decode('hex') + assert self.rfile.read(9) == codecs.decode('000000040100000000', 'hex_codec') self.wfile.write("OK") self.wfile.flush() self.rfile.safe_read(9) # just to keep the connection alive a bit longer @@ -223,7 +218,7 @@ class TestApplySettings(tservers.ServerTestBase): SettingsFrame.INITIAL_WINDOW_SIZE: 'deadbeef', }) - assert c.rfile.safe_read(2) == "OK" + assert c.rfile.safe_read(2) == b"OK" assert protocol.http2_settings[ SettingsFrame.ENABLE_PUSH] == 'foo' @@ -233,7 +228,7 @@ class TestApplySettings(tservers.ServerTestBase): SettingsFrame.INITIAL_WINDOW_SIZE] == 'deadbeef' -class TestCreateHeaders(): +class TestCreateHeaders(object): c = tcp.TCPClient(("127.0.0.1", 0)) def test_create_headers(self): @@ -246,14 +241,12 @@ class TestCreateHeaders(): bytes = HTTP2Protocol(self.c)._create_headers( headers, 1, end_stream=True) assert b''.join(bytes) ==\ - '000014010500000001824488355217caf3a69a3f87408294e7838c767f'\ - .decode('hex') + codecs.decode('000014010500000001824488355217caf3a69a3f87408294e7838c767f', 'hex_codec') bytes = HTTP2Protocol(self.c)._create_headers( headers, 1, end_stream=False) assert b''.join(bytes) ==\ - '000014010400000001824488355217caf3a69a3f87408294e7838c767f'\ - .decode('hex') + codecs.decode('000014010400000001824488355217caf3a69a3f87408294e7838c767f', 'hex_codec') def test_create_headers_multiple_frames(self): headers = http.Headers([ @@ -267,41 +260,42 @@ class TestCreateHeaders(): protocol.http2_settings[SettingsFrame.MAX_FRAME_SIZE] = 8 bytes = protocol._create_headers(headers, 1, end_stream=True) assert len(bytes) == 3 - assert bytes[0] == '000008010100000001828487408294e783'.decode('hex') - assert bytes[1] == '0000080900000000018c767f7685ee5b10'.decode('hex') - assert bytes[2] == '00000209040000000163d5'.decode('hex') + assert bytes[0] == codecs.decode('000008010100000001828487408294e783', 'hex_codec') + assert bytes[1] == codecs.decode('0000080900000000018c767f7685ee5b10', 'hex_codec') + assert bytes[2] == codecs.decode('00000209040000000163d5', 'hex_codec') -class TestCreateBody(): +class TestCreateBody(object): c = tcp.TCPClient(("127.0.0.1", 0)) def test_create_body_empty(self): protocol = HTTP2Protocol(self.c) bytes = protocol._create_body(b'', 1) - assert b''.join(bytes) == ''.decode('hex') + assert b''.join(bytes) == b'' def test_create_body_single_frame(self): protocol = HTTP2Protocol(self.c) bytes = protocol._create_body(b'foobar', 1) - assert b''.join(bytes) == '000006000100000001666f6f626172'.decode('hex') + assert b''.join(bytes) == codecs.decode('000006000100000001666f6f626172', 'hex_codec') def test_create_body_multiple_frames(self): protocol = HTTP2Protocol(self.c) protocol.http2_settings[SettingsFrame.MAX_FRAME_SIZE] = 5 bytes = protocol._create_body(b'foobarmehm42', 1) assert len(bytes) == 3 - assert bytes[0] == '000005000000000001666f6f6261'.decode('hex') - assert bytes[1] == '000005000000000001726d65686d'.decode('hex') - assert bytes[2] == '0000020001000000013432'.decode('hex') + assert bytes[0] == codecs.decode('000005000000000001666f6f6261', 'hex_codec') + assert bytes[1] == codecs.decode('000005000000000001726d65686d', 'hex_codec') + assert bytes[2] == codecs.decode('0000020001000000013432', 'hex_codec') class TestReadRequest(tservers.ServerTestBase): class handler(tcp.BaseHandler): + def handle(self): self.wfile.write( - b'000003010400000001828487'.decode('hex')) + codecs.decode('000003010400000001828487', 'hex_codec')) self.wfile.write( - b'000006000100000001666f6f626172'.decode('hex')) + codecs.decode('000006000100000001666f6f626172', 'hex_codec')) self.wfile.flush() self.rfile.safe_read(9) # just to keep the connection alive a bit longer @@ -317,7 +311,7 @@ class TestReadRequest(tservers.ServerTestBase): req = protocol.read_request(NotImplemented) assert req.stream_id - assert req.headers.fields == [[':method', 'GET'], [':path', '/'], [':scheme', 'https']] + assert req.headers.fields == [[b':method', b'GET'], [b':path', b'/'], [b':scheme', b'https']] assert req.content == b'foobar' @@ -325,7 +319,7 @@ class TestReadRequestRelative(tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): self.wfile.write( - b'00000c0105000000014287d5af7e4d5a777f4481f9'.decode('hex')) + codecs.decode('00000c0105000000014287d5af7e4d5a777f4481f9', 'hex_codec')) self.wfile.flush() ssl = True @@ -348,7 +342,7 @@ class TestReadRequestAbsolute(tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): self.wfile.write( - b'00001901050000000182448d9d29aee30c0e492c2a1170426366871c92585422e085'.decode('hex')) + codecs.decode('00001901050000000182448d9d29aee30c0e492c2a1170426366871c92585422e085', 'hex_codec')) self.wfile.flush() ssl = True @@ -372,9 +366,9 @@ class TestReadRequestConnect(tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): self.wfile.write( - b'00001b0105000000014287bdab4e9c17b7ff44871c92585422e08541871c92585422e085'.decode('hex')) + codecs.decode('00001b0105000000014287bdab4e9c17b7ff44871c92585422e08541871c92585422e085', 'hex_codec')) self.wfile.write( - b'00001d0105000000014287bdab4e9c17b7ff44882f91d35d055c87a741882f91d35d055c87a7'.decode('hex')) + codecs.decode('00001d0105000000014287bdab4e9c17b7ff44882f91d35d055c87a741882f91d35d055c87a7', 'hex_codec')) self.wfile.flush() ssl = True @@ -403,9 +397,9 @@ class TestReadResponse(tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): self.wfile.write( - b'00000801040000002a88628594e78c767f'.decode('hex')) + codecs.decode('00000801040000002a88628594e78c767f', 'hex_codec')) self.wfile.write( - b'00000600010000002a666f6f626172'.decode('hex')) + codecs.decode('00000600010000002a666f6f626172', 'hex_codec')) self.wfile.flush() self.rfile.safe_read(9) # just to keep the connection alive a bit longer @@ -420,10 +414,10 @@ class TestReadResponse(tservers.ServerTestBase): resp = protocol.read_response(NotImplemented, stream_id=42) - assert resp.http_version == (2, 0) + assert resp.http_version == '2.0' assert resp.status_code == 200 - assert resp.msg == "" - assert resp.headers.fields == [[':status', '200'], ['etag', 'foobar']] + assert resp.msg == '' + assert resp.headers.fields == [[b':status', b'200'], [b'etag', b'foobar']] assert resp.content == b'foobar' assert resp.timestamp_end @@ -432,7 +426,7 @@ class TestReadEmptyResponse(tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): self.wfile.write( - b'00000801050000002a88628594e78c767f'.decode('hex')) + codecs.decode('00000801050000002a88628594e78c767f', 'hex_codec')) self.wfile.flush() ssl = True @@ -447,10 +441,10 @@ class TestReadEmptyResponse(tservers.ServerTestBase): resp = protocol.read_response(NotImplemented, stream_id=42) assert resp.stream_id == 42 - assert resp.http_version == (2, 0) + assert resp.http_version == '2.0' assert resp.status_code == 200 - assert resp.msg == b'' - assert resp.headers.fields == [[':status', '200'], ['etag', 'foobar']] + assert resp.msg == '' + assert resp.headers.fields == [[b':status', b'200'], [b'etag', b'foobar']] assert resp.content == b'' @@ -459,53 +453,53 @@ class TestAssembleRequest(object): def test_request_simple(self): bytes = HTTP2Protocol(self.c).assemble_request(http.Request( - '', - 'GET', - 'https', - '', - '', - '/', - (2, 0), + b'', + b'GET', + b'https', + b'', + b'', + b'/', + b'2.0', None, None, )) assert len(bytes) == 1 - assert bytes[0] == '00000d0105000000018284874188089d5c0b8170dc07'.decode('hex') + assert bytes[0] == codecs.decode('00000d0105000000018284874188089d5c0b8170dc07', 'hex_codec') def test_request_with_stream_id(self): req = http.Request( - '', - 'GET', - 'https', - '', - '', - '/', - (2, 0), + b'', + b'GET', + b'https', + b'', + b'', + b'/', + b'2.0', None, None, ) req.stream_id = 0x42 bytes = HTTP2Protocol(self.c).assemble_request(req) assert len(bytes) == 1 - assert bytes[0] == '00000d0105000000428284874188089d5c0b8170dc07'.decode('hex') + assert bytes[0] == codecs.decode('00000d0105000000428284874188089d5c0b8170dc07', 'hex_codec') def test_request_with_body(self): bytes = HTTP2Protocol(self.c).assemble_request(http.Request( - '', - 'GET', - 'https', - '', - '', - '/', - (2, 0), - http.Headers([('foo', 'bar')]), - 'foobar', + b'', + b'GET', + b'https', + b'', + b'', + b'/', + b'2.0', + http.Headers([(b'foo', b'bar')]), + b'foobar', )) assert len(bytes) == 2 assert bytes[0] ==\ - '0000150104000000018284874188089d5c0b8170dc07408294e7838c767f'.decode('hex') + codecs.decode('0000150104000000018284874188089d5c0b8170dc07408294e7838c767f', 'hex_codec') assert bytes[1] ==\ - '000006000100000001666f6f626172'.decode('hex') + codecs.decode('000006000100000001666f6f626172', 'hex_codec') class TestAssembleResponse(object): @@ -513,34 +507,34 @@ class TestAssembleResponse(object): def test_simple(self): bytes = HTTP2Protocol(self.c, is_server=True).assemble_response(http.Response( - (2, 0), + b'2.0', 200, )) assert len(bytes) == 1 assert bytes[0] ==\ - '00000101050000000288'.decode('hex') + codecs.decode('00000101050000000288', 'hex_codec') def test_with_stream_id(self): resp = http.Response( - (2, 0), + b'2.0', 200, ) resp.stream_id = 0x42 bytes = HTTP2Protocol(self.c, is_server=True).assemble_response(resp) assert len(bytes) == 1 assert bytes[0] ==\ - '00000101050000004288'.decode('hex') + codecs.decode('00000101050000004288', 'hex_codec') def test_with_body(self): bytes = HTTP2Protocol(self.c, is_server=True).assemble_response(http.Response( - (2, 0), + b'2.0', 200, - '', - Headers(foo="bar"), - 'foobar' + b'', + http.Headers(foo=b"bar"), + b'foobar' )) assert len(bytes) == 2 assert bytes[0] ==\ - '00000901040000000288408294e7838c767f'.decode('hex') + codecs.decode('00000901040000000288408294e7838c767f', 'hex_codec') assert bytes[1] ==\ - '000006000100000002666f6f626172'.decode('hex') + codecs.decode('000006000100000002666f6f626172', 'hex_codec')