From d3611777539471e53d4fdedf352ed755a4092415 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sun, 3 Jul 2016 18:03:34 +0530 Subject: [PATCH] h2: move header parsing to netlib --- mitmproxy/protocol/http2.py | 30 ++--------------------- netlib/http/http2/__init__.py | 2 ++ netlib/http/http2/utils.py | 37 ++++++++++++++++++++++++++++ pathod/protocols/http2.py | 46 ++++++----------------------------- 4 files changed, 48 insertions(+), 67 deletions(-) create mode 100644 netlib/http/http2/utils.py diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index b9a30c7e1..f6261b6b6 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -330,39 +330,13 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) if self.zombie: # pragma: no cover raise exceptions.Http2ProtocolException("Zombie Stream") - authority = self.request_headers.get(':authority', '') - method = self.request_headers.get(':method', 'GET') - scheme = self.request_headers.get(':scheme', 'https') - path = self.request_headers.get(':path', '/') - self.request_headers.clear(":method") - self.request_headers.clear(":scheme") - self.request_headers.clear(":path") - host = None - port = None - - if path == '*' or path.startswith("/"): - first_line_format = "relative" - elif method == 'CONNECT': # pragma: no cover - raise NotImplementedError("CONNECT over HTTP/2 is not implemented.") - else: # pragma: no cover - first_line_format = "absolute" - # FIXME: verify if path or :host contains what we need - scheme, host, port, _ = netlib.http.url.parse(path) - - if authority: - host, _, port = authority.partition(':') - - if not host: - host = 'localhost' - if not port: - port = 443 if scheme == 'https' else 80 - port = int(port) - data = [] while self.request_data_queue.qsize() > 0: data.append(self.request_data_queue.get()) data = b"".join(data) + first_line_format, method, scheme, host, port, path = http2.parse_headers(self.request_headers) + return models.HTTPRequest( first_line_format, method, diff --git a/netlib/http/http2/__init__.py b/netlib/http/http2/__init__.py index 6a979a0d0..600641906 100644 --- a/netlib/http/http2/__init__.py +++ b/netlib/http/http2/__init__.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division from netlib.http.http2 import framereader +from netlib.http.http2.utils import parse_headers __all__ = [ "framereader", + "parse_headers", ] diff --git a/netlib/http/http2/utils.py b/netlib/http/http2/utils.py new file mode 100644 index 000000000..4d5f580cd --- /dev/null +++ b/netlib/http/http2/utils.py @@ -0,0 +1,37 @@ +from netlib.http import url + + +def parse_headers(headers): + authority = headers.get(':authority', b'') + method = headers.get(':method', b'GET') + scheme = headers.get(':scheme', b'https') + path = headers.get(':path', b'/') + + headers.clear(":method") + headers.clear(":scheme") + headers.clear(":path") + + host = None + port = None + + if path == b'*' or path.startswith(b"/"): + first_line_format = "relative" + elif method == b'CONNECT': # pragma: no cover + raise NotImplementedError("CONNECT over HTTP/2 is not implemented.") + else: # pragma: no cover + first_line_format = "absolute" + # FIXME: verify if path or :host contains what we need + scheme, host, port, _ = url.parse(path) + + if authority: + host, _, port = authority.partition(b':') + + if not host: + host = b'localhost' + + if not port: + port = 443 if scheme == b'https' else 80 + + port = int(port) + + return first_line_format, method, scheme, host, port, path diff --git a/pathod/protocols/http2.py b/pathod/protocols/http2.py index c87289406..d94cd9815 100644 --- a/pathod/protocols/http2.py +++ b/pathod/protocols/http2.py @@ -7,8 +7,7 @@ import hyperframe.frame from hpack.hpack import Encoder, Decoder from netlib import utils, strutils -from netlib.http import url -from netlib.http.http2 import framereader +from netlib.http import http2 import netlib.http.headers import netlib.http.response import netlib.http.request @@ -101,46 +100,15 @@ class HTTP2StateProtocol(object): timestamp_end = time.time() - authority = headers.get(':authority', b'') - method = headers.get(':method', 'GET') - scheme = headers.get(':scheme', 'https') - path = headers.get(':path', '/') - - headers.clear(":method") - headers.clear(":scheme") - headers.clear(":path") - - host = None - port = None - - if path == '*' or path.startswith("/"): - first_line_format = "relative" - elif method == 'CONNECT': - first_line_format = "authority" - if ":" in authority: - host, port = authority.split(":", 1) - else: - host = authority - else: - first_line_format = "absolute" - # FIXME: verify if path or :host contains what we need - scheme, host, port, _ = url.parse(path) - scheme = scheme.decode('ascii') - host = host.decode('ascii') - - if host is None: - host = 'localhost' - if port is None: - port = 80 if scheme == 'http' else 443 - port = int(port) + first_line_format, method, scheme, host, port, path = http2.parse_headers(headers) request = netlib.http.request.Request( first_line_format, - method.encode('ascii'), - scheme.encode('ascii'), - host.encode('ascii'), + method, + scheme, + host, port, - path.encode('ascii'), + path, b"HTTP/2.0", headers, body, @@ -286,7 +254,7 @@ class HTTP2StateProtocol(object): def read_frame(self, hide=False): while True: - frm = framereader.http2_read_frame(self.tcp_handler.rfile) + frm = http2.framereader.http2_read_frame(self.tcp_handler.rfile) if not hide and self.dump_frames: # pragma no cover print(frm.human_readable("<<"))