diff --git a/netlib/http/http1/protocol.py b/netlib/http/http1/protocol.py index d2a77399c..e7727e00c 100644 --- a/netlib/http/http1/protocol.py +++ b/netlib/http/http1/protocol.py @@ -15,15 +15,144 @@ class HTTP1Protocol(object): self.tcp_handler = tcp_handler - def get_request_line(self): + def read_request(self, include_body=True, body_size_limit=None, allow_empty=False): """ - Get a line, possibly preceded by a blank. + Parse an HTTP request from a file stream + + Args: + include_body (bool): Read response body as well + body_size_limit (bool): Maximum body size + wfile (file): If specified, HTTP Expect headers are handled + automatically, by writing a HTTP 100 CONTINUE response to the stream. + + Returns: + Request: The HTTP request + + Raises: + HttpError: If the input is invalid. """ + httpversion, host, port, scheme, method, path, headers, content = ( + None, None, None, None, None, None, None, None) + + request_line = self._get_request_line() + if not request_line: + if allow_empty: + return http.EmptyRequest() + else: + raise tcp.NetLibDisconnect() + + request_line_parts = self._parse_init(request_line) + if not request_line_parts: + raise HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) + method, path, httpversion = request_line_parts + + if path == '*' or path.startswith("/"): + form_in = "relative" + if not utils.isascii(path): + raise HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) + elif method.upper() == 'CONNECT': + form_in = "authority" + r = self._parse_init_connect(request_line) + if not r: + raise HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) + host, port, _ = r + return http.ConnectRequest(host, port) + else: + form_in = "absolute" + r = self._parse_init_proxy(request_line) + if not r: + raise HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) + _, scheme, host, port, path, _ = r + + headers = self.read_headers() + if headers is None: + raise HttpError(400, "Invalid headers") + + expect_header = headers.get_first("expect", "").lower() + if expect_header == "100-continue" and httpversion >= (1, 1): + self.tcp_handler.wfile.write( + 'HTTP/1.1 100 Continue\r\n' + '\r\n' + ) + self.tcp_handler.wfile.flush() + del headers['expect'] + + if include_body: + content = self.read_http_body( + headers, + body_size_limit, + method, + None, + True + ) + + return http.Request( + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + content + ) + + + def read_response(self, request_method, body_size_limit, include_body=True): + """ + Returns an http.Response + + By default, both response header and body are read. + If include_body=False is specified, content may be one of the + following: + - None, if the response is technically allowed to have a response body + - "", if the response must not have a response body (e.g. it's a + response to a HEAD request) + """ + line = self.tcp_handler.rfile.readline() + # Possible leftover from previous message if line == "\r\n" or line == "\n": - # Possible leftover from previous message line = self.tcp_handler.rfile.readline() - return line + if not line: + raise HttpErrorConnClosed(502, "Server disconnect.") + parts = self.parse_response_line(line) + if not parts: + raise HttpError(502, "Invalid server response: %s" % repr(line)) + proto, code, msg = parts + httpversion = self._parse_http_protocol(proto) + if httpversion is None: + raise HttpError(502, "Invalid HTTP version in line: %s" % repr(proto)) + headers = self.read_headers() + if headers is None: + raise HttpError(502, "Invalid headers.") + + if include_body: + content = self.read_http_body( + headers, + body_size_limit, + request_method, + code, + False + ) + else: + # if include_body==False then a None content means the body should be + # read separately + content = None + return http.Response(httpversion, code, msg, headers, content) def read_headers(self): @@ -56,177 +185,6 @@ class HTTP1Protocol(object): return odict.ODictCaseless(ret) - def read_chunked(self, limit, is_request): - """ - Read a chunked HTTP body. - - May raise HttpError. - """ - # FIXME: Should check if chunked is the final encoding in the headers - # http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-16#section-3.3 - # 3.3 2. - total = 0 - code = 400 if is_request else 502 - while True: - line = self.tcp_handler.rfile.readline(128) - if line == "": - raise HttpErrorConnClosed(code, "Connection closed prematurely") - if line != '\r\n' and line != '\n': - try: - length = int(line, 16) - except ValueError: - raise HttpError( - code, - "Invalid chunked encoding length: %s" % line - ) - total += length - if limit is not None and total > limit: - msg = "HTTP Body too large. Limit is %s," \ - " chunked content longer than %s" % (limit, total) - raise HttpError(code, msg) - chunk = self.tcp_handler.rfile.read(length) - suffix = self.tcp_handler.rfile.readline(5) - if suffix != '\r\n': - raise HttpError(code, "Malformed chunked body") - yield line, chunk, '\r\n' - if length == 0: - return - - - @classmethod - def has_chunked_encoding(self, headers): - return "chunked" in [ - i.lower() for i in http.get_header_tokens(headers, "transfer-encoding") - ] - - - @classmethod - def parse_http_protocol(self, line): - """ - Parse an HTTP protocol declaration. - Returns a (major, minor) tuple, or None. - """ - if not line.startswith("HTTP/"): - return None - _, version = line.split('/', 1) - if "." not in version: - return None - major, minor = version.split('.', 1) - try: - major = int(major) - minor = int(minor) - except ValueError: - return None - return major, minor - - - @classmethod - def parse_init(self, line): - try: - method, url, protocol = string.split(line) - except ValueError: - return None - httpversion = self.parse_http_protocol(protocol) - if not httpversion: - return None - if not utils.isascii(method): - return None - return method, url, httpversion - - - @classmethod - def parse_init_connect(self, line): - """ - Returns (host, port, httpversion) if line is a valid CONNECT line. - http://tools.ietf.org/html/draft-luotonen-web-proxy-tunneling-01 section 3.1 - """ - v = self.parse_init(line) - if not v: - return None - method, url, httpversion = v - - if method.upper() != 'CONNECT': - return None - try: - host, port = url.split(":") - except ValueError: - return None - try: - port = int(port) - except ValueError: - return None - if not http.is_valid_port(port): - return None - if not http.is_valid_host(host): - return None - return host, port, httpversion - - - @classmethod - def parse_init_proxy(self, line): - v = self.parse_init(line) - if not v: - return None - method, url, httpversion = v - - parts = http.parse_url(url) - if not parts: - return None - scheme, host, port, path = parts - return method, scheme, host, port, path, httpversion - - - @classmethod - def parse_init_http(self, line): - """ - Returns (method, url, httpversion) - """ - v = self.parse_init(line) - if not v: - return None - method, url, httpversion = v - if not utils.isascii(url): - return None - if not (url.startswith("/") or url == "*"): - return None - return method, url, httpversion - - - @classmethod - def connection_close(self, httpversion, headers): - """ - Checks the message to see if the client connection should be closed - according to RFC 2616 Section 8.1 Note that a connection should be - closed as well if the response has been read until end of the stream. - """ - # At first, check if we have an explicit Connection header. - if "connection" in headers: - toks = http.get_header_tokens(headers, "connection") - if "close" in toks: - return True - elif "keep-alive" in toks: - return False - - # If we don't have a Connection header, HTTP 1.1 connections are assumed to - # be persistent - return httpversion != (1, 1) - - - @classmethod - def parse_response_line(self, line): - parts = line.strip().split(" ", 2) - if len(parts) == 2: # handle missing message gracefully - parts.append("") - if len(parts) != 3: - return None - proto, code, msg = parts - try: - code = int(code) - except ValueError: - return None - return (proto, code, msg) - - def read_http_body(self, *args, **kwargs): return "".join( content for _, content, _ in self.read_http_body_chunked(*args, **kwargs) @@ -241,7 +199,7 @@ class HTTP1Protocol(object): response_code, is_request, max_chunk_size=None - ): + ): """ Read an HTTP message body: headers: An ODictCaseless object @@ -259,7 +217,7 @@ class HTTP1Protocol(object): if expected_size is None: if self.has_chunked_encoding(headers): # Python 3: yield from - for x in self.read_chunked(limit, is_request): + for x in self._read_chunked(limit, is_request): yield x else: # pragma: nocover raise HttpError( @@ -333,146 +291,6 @@ class HTTP1Protocol(object): return -1 - def read_request(self, include_body=True, body_size_limit=None, allow_empty=False): - """ - Parse an HTTP request from a file stream - - Args: - include_body (bool): Read response body as well - body_size_limit (bool): Maximum body size - wfile (file): If specified, HTTP Expect headers are handled - automatically, by writing a HTTP 100 CONTINUE response to the stream. - - Returns: - Request: The HTTP request - - Raises: - HttpError: If the input is invalid. - """ - httpversion, host, port, scheme, method, path, headers, content = ( - None, None, None, None, None, None, None, None) - - request_line = self.get_request_line() - if not request_line: - if allow_empty: - return http.EmptyRequest() - else: - raise tcp.NetLibDisconnect() - - request_line_parts = self.parse_init(request_line) - if not request_line_parts: - raise HttpError( - 400, - "Bad HTTP request line: %s" % repr(request_line) - ) - method, path, httpversion = request_line_parts - - if path == '*' or path.startswith("/"): - form_in = "relative" - if not utils.isascii(path): - raise HttpError( - 400, - "Bad HTTP request line: %s" % repr(request_line) - ) - elif method.upper() == 'CONNECT': - form_in = "authority" - r = self.parse_init_connect(request_line) - if not r: - raise HttpError( - 400, - "Bad HTTP request line: %s" % repr(request_line) - ) - host, port, _ = r - return http.ConnectRequest(host, port) - else: - form_in = "absolute" - r = self.parse_init_proxy(request_line) - if not r: - raise HttpError( - 400, - "Bad HTTP request line: %s" % repr(request_line) - ) - _, scheme, host, port, path, _ = r - - headers = self.read_headers() - if headers is None: - raise HttpError(400, "Invalid headers") - - expect_header = headers.get_first("expect", "").lower() - if expect_header == "100-continue" and httpversion >= (1, 1): - self.tcp_handler.wfile.write( - 'HTTP/1.1 100 Continue\r\n' - '\r\n' - ) - self.tcp_handler.wfile.flush() - del headers['expect'] - - if include_body: - content = self.read_http_body( - headers, - body_size_limit, - method, - None, - True - ) - - return http.Request( - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - content - ) - - - def read_response(self, request_method, body_size_limit, include_body=True): - """ - Returns an http.Response - - By default, both response header and body are read. - If include_body=False is specified, content may be one of the - following: - - None, if the response is technically allowed to have a response body - - "", if the response must not have a response body (e.g. it's a - response to a HEAD request) - """ - - line = self.tcp_handler.rfile.readline() - # Possible leftover from previous message - if line == "\r\n" or line == "\n": - line = self.tcp_handler.rfile.readline() - if not line: - raise HttpErrorConnClosed(502, "Server disconnect.") - parts = self.parse_response_line(line) - if not parts: - raise HttpError(502, "Invalid server response: %s" % repr(line)) - proto, code, msg = parts - httpversion = self.parse_http_protocol(proto) - if httpversion is None: - raise HttpError(502, "Invalid HTTP version in line: %s" % repr(proto)) - headers = self.read_headers() - if headers is None: - raise HttpError(502, "Invalid headers.") - - if include_body: - content = self.read_http_body( - headers, - body_size_limit, - request_method, - code, - False - ) - else: - # if include_body==False then a None content means the body should be - # read separately - content = None - return http.Response(httpversion, code, msg, headers, content) - - @classmethod def request_preamble(self, method, resource, http_major="1", http_minor="1"): return '%s %s HTTP/%s.%s' % ( @@ -485,3 +303,186 @@ class HTTP1Protocol(object): if message is None: message = status_codes.RESPONSES.get(code) return 'HTTP/%s.%s %s %s' % (http_major, http_minor, code, message) + + + @classmethod + def has_chunked_encoding(self, headers): + return "chunked" in [ + i.lower() for i in http.get_header_tokens(headers, "transfer-encoding") + ] + + + def _get_request_line(self): + """ + Get a line, possibly preceded by a blank. + """ + line = self.tcp_handler.rfile.readline() + if line == "\r\n" or line == "\n": + # Possible leftover from previous message + line = self.tcp_handler.rfile.readline() + return line + + + + def _read_chunked(self, limit, is_request): + """ + Read a chunked HTTP body. + + May raise HttpError. + """ + # FIXME: Should check if chunked is the final encoding in the headers + # http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-16#section-3.3 + # 3.3 2. + total = 0 + code = 400 if is_request else 502 + while True: + line = self.tcp_handler.rfile.readline(128) + if line == "": + raise HttpErrorConnClosed(code, "Connection closed prematurely") + if line != '\r\n' and line != '\n': + try: + length = int(line, 16) + except ValueError: + raise HttpError( + code, + "Invalid chunked encoding length: %s" % line + ) + total += length + if limit is not None and total > limit: + msg = "HTTP Body too large. Limit is %s," \ + " chunked content longer than %s" % (limit, total) + raise HttpError(code, msg) + chunk = self.tcp_handler.rfile.read(length) + suffix = self.tcp_handler.rfile.readline(5) + if suffix != '\r\n': + raise HttpError(code, "Malformed chunked body") + yield line, chunk, '\r\n' + if length == 0: + return + + + @classmethod + def _parse_http_protocol(self, line): + """ + Parse an HTTP protocol declaration. + Returns a (major, minor) tuple, or None. + """ + if not line.startswith("HTTP/"): + return None + _, version = line.split('/', 1) + if "." not in version: + return None + major, minor = version.split('.', 1) + try: + major = int(major) + minor = int(minor) + except ValueError: + return None + return major, minor + + + @classmethod + def _parse_init(self, line): + try: + method, url, protocol = string.split(line) + except ValueError: + return None + httpversion = self._parse_http_protocol(protocol) + if not httpversion: + return None + if not utils.isascii(method): + return None + return method, url, httpversion + + + @classmethod + def _parse_init_connect(self, line): + """ + Returns (host, port, httpversion) if line is a valid CONNECT line. + http://tools.ietf.org/html/draft-luotonen-web-proxy-tunneling-01 section 3.1 + """ + v = self._parse_init(line) + if not v: + return None + method, url, httpversion = v + + if method.upper() != 'CONNECT': + return None + try: + host, port = url.split(":") + except ValueError: + return None + try: + port = int(port) + except ValueError: + return None + if not http.is_valid_port(port): + return None + if not http.is_valid_host(host): + return None + return host, port, httpversion + + + @classmethod + def _parse_init_proxy(self, line): + v = self._parse_init(line) + if not v: + return None + method, url, httpversion = v + + parts = http.parse_url(url) + if not parts: + return None + scheme, host, port, path = parts + return method, scheme, host, port, path, httpversion + + + @classmethod + def _parse_init_http(self, line): + """ + Returns (method, url, httpversion) + """ + v = self._parse_init(line) + if not v: + return None + method, url, httpversion = v + if not utils.isascii(url): + return None + if not (url.startswith("/") or url == "*"): + return None + return method, url, httpversion + + + @classmethod + def connection_close(self, httpversion, headers): + """ + Checks the message to see if the client connection should be closed + according to RFC 2616 Section 8.1 Note that a connection should be + closed as well if the response has been read until end of the stream. + """ + # At first, check if we have an explicit Connection header. + if "connection" in headers: + toks = http.get_header_tokens(headers, "connection") + if "close" in toks: + return True + elif "keep-alive" in toks: + return False + + # If we don't have a Connection header, HTTP 1.1 connections are assumed to + # be persistent + return httpversion != (1, 1) + + + @classmethod + def parse_response_line(self, line): + parts = line.strip().split(" ", 2) + if len(parts) == 2: # handle missing message gracefully + parts.append("") + if len(parts) != 3: + return None + proto, code, msg = parts + try: + code = int(code) + except ValueError: + return None + return (proto, code, msg) diff --git a/test/http/http1/test_protocol.py b/test/http/http1/test_protocol.py index 936fe20de..8d05b31fc 100644 --- a/test/http/http1/test_protocol.py +++ b/test/http/http1/test_protocol.py @@ -181,29 +181,29 @@ def test_expected_http_body_size(): def test_parse_http_protocol(): - assert HTTP1Protocol.parse_http_protocol("HTTP/1.1") == (1, 1) - assert HTTP1Protocol.parse_http_protocol("HTTP/0.0") == (0, 0) - assert not HTTP1Protocol.parse_http_protocol("HTTP/a.1") - assert not HTTP1Protocol.parse_http_protocol("HTTP/1.a") - assert not HTTP1Protocol.parse_http_protocol("foo/0.0") - assert not HTTP1Protocol.parse_http_protocol("HTTP/x") + assert HTTP1Protocol._parse_http_protocol("HTTP/1.1") == (1, 1) + assert HTTP1Protocol._parse_http_protocol("HTTP/0.0") == (0, 0) + assert not HTTP1Protocol._parse_http_protocol("HTTP/a.1") + assert not HTTP1Protocol._parse_http_protocol("HTTP/1.a") + assert not HTTP1Protocol._parse_http_protocol("foo/0.0") + assert not HTTP1Protocol._parse_http_protocol("HTTP/x") def test_parse_init_connect(): - assert HTTP1Protocol.parse_init_connect("CONNECT host.com:443 HTTP/1.0") - assert not HTTP1Protocol.parse_init_connect("C\xfeONNECT host.com:443 HTTP/1.0") - assert not HTTP1Protocol.parse_init_connect("CONNECT \0host.com:443 HTTP/1.0") - assert not HTTP1Protocol.parse_init_connect("CONNECT host.com:444444 HTTP/1.0") - assert not HTTP1Protocol.parse_init_connect("bogus") - assert not HTTP1Protocol.parse_init_connect("GET host.com:443 HTTP/1.0") - assert not HTTP1Protocol.parse_init_connect("CONNECT host.com443 HTTP/1.0") - assert not HTTP1Protocol.parse_init_connect("CONNECT host.com:443 foo/1.0") - assert not HTTP1Protocol.parse_init_connect("CONNECT host.com:foo HTTP/1.0") + assert HTTP1Protocol._parse_init_connect("CONNECT host.com:443 HTTP/1.0") + assert not HTTP1Protocol._parse_init_connect("C\xfeONNECT host.com:443 HTTP/1.0") + assert not HTTP1Protocol._parse_init_connect("CONNECT \0host.com:443 HTTP/1.0") + assert not HTTP1Protocol._parse_init_connect("CONNECT host.com:444444 HTTP/1.0") + assert not HTTP1Protocol._parse_init_connect("bogus") + assert not HTTP1Protocol._parse_init_connect("GET host.com:443 HTTP/1.0") + assert not HTTP1Protocol._parse_init_connect("CONNECT host.com443 HTTP/1.0") + assert not HTTP1Protocol._parse_init_connect("CONNECT host.com:443 foo/1.0") + assert not HTTP1Protocol._parse_init_connect("CONNECT host.com:foo HTTP/1.0") def test_parse_init_proxy(): u = "GET http://foo.com:8888/test HTTP/1.1" - m, s, h, po, pa, httpversion = HTTP1Protocol.parse_init_proxy(u) + m, s, h, po, pa, httpversion = HTTP1Protocol._parse_init_proxy(u) assert m == "GET" assert s == "http" assert h == "foo.com" @@ -212,27 +212,27 @@ def test_parse_init_proxy(): assert httpversion == (1, 1) u = "G\xfeET http://foo.com:8888/test HTTP/1.1" - assert not HTTP1Protocol.parse_init_proxy(u) + assert not HTTP1Protocol._parse_init_proxy(u) - assert not HTTP1Protocol.parse_init_proxy("invalid") - assert not HTTP1Protocol.parse_init_proxy("GET invalid HTTP/1.1") - assert not HTTP1Protocol.parse_init_proxy("GET http://foo.com:8888/test foo/1.1") + assert not HTTP1Protocol._parse_init_proxy("invalid") + assert not HTTP1Protocol._parse_init_proxy("GET invalid HTTP/1.1") + assert not HTTP1Protocol._parse_init_proxy("GET http://foo.com:8888/test foo/1.1") def test_parse_init_http(): u = "GET /test HTTP/1.1" - m, u, httpversion = HTTP1Protocol.parse_init_http(u) + m, u, httpversion = HTTP1Protocol._parse_init_http(u) assert m == "GET" assert u == "/test" assert httpversion == (1, 1) u = "G\xfeET /test HTTP/1.1" - assert not HTTP1Protocol.parse_init_http(u) + assert not HTTP1Protocol._parse_init_http(u) - assert not HTTP1Protocol.parse_init_http("invalid") - assert not HTTP1Protocol.parse_init_http("GET invalid HTTP/1.1") - assert not HTTP1Protocol.parse_init_http("GET /test foo/1.1") - assert not HTTP1Protocol.parse_init_http("GET /test\xc0 HTTP/1.1") + assert not HTTP1Protocol._parse_init_http("invalid") + assert not HTTP1Protocol._parse_init_http("GET invalid HTTP/1.1") + assert not HTTP1Protocol._parse_init_http("GET /test foo/1.1") + assert not HTTP1Protocol._parse_init_http("GET /test\xc0 HTTP/1.1") class TestReadHeaders: @@ -367,8 +367,8 @@ def test_read_response(): def test_get_request_line(): data = "\nfoo" p = mock_protocol(data) - assert p.get_request_line() == "foo" - assert not p.get_request_line() + assert p._get_request_line() == "foo" + assert not p._get_request_line() class TestReadRequest():