reduce public interface

use private indicator pattern “_methodname”
This commit is contained in:
Thomas Kriechbaumer 2015-07-19 23:25:15 +02:00
parent 83f013fca1
commit ecc7ffe928
2 changed files with 346 additions and 345 deletions

View File

@ -15,15 +15,144 @@ class HTTP1Protocol(object):
self.tcp_handler = tcp_handler
def get_request_line(self):
def read_request(self, include_body=True, body_size_limit=None, allow_empty=False):
"""
Get a line, possibly preceded by a blank.
Parse an HTTP request from a file stream
Args:
include_body (bool): Read response body as well
body_size_limit (bool): Maximum body size
wfile (file): If specified, HTTP Expect headers are handled
automatically, by writing a HTTP 100 CONTINUE response to the stream.
Returns:
Request: The HTTP request
Raises:
HttpError: If the input is invalid.
"""
httpversion, host, port, scheme, method, path, headers, content = (
None, None, None, None, None, None, None, None)
request_line = self._get_request_line()
if not request_line:
if allow_empty:
return http.EmptyRequest()
else:
raise tcp.NetLibDisconnect()
request_line_parts = self._parse_init(request_line)
if not request_line_parts:
raise HttpError(
400,
"Bad HTTP request line: %s" % repr(request_line)
)
method, path, httpversion = request_line_parts
if path == '*' or path.startswith("/"):
form_in = "relative"
if not utils.isascii(path):
raise HttpError(
400,
"Bad HTTP request line: %s" % repr(request_line)
)
elif method.upper() == 'CONNECT':
form_in = "authority"
r = self._parse_init_connect(request_line)
if not r:
raise HttpError(
400,
"Bad HTTP request line: %s" % repr(request_line)
)
host, port, _ = r
return http.ConnectRequest(host, port)
else:
form_in = "absolute"
r = self._parse_init_proxy(request_line)
if not r:
raise HttpError(
400,
"Bad HTTP request line: %s" % repr(request_line)
)
_, scheme, host, port, path, _ = r
headers = self.read_headers()
if headers is None:
raise HttpError(400, "Invalid headers")
expect_header = headers.get_first("expect", "").lower()
if expect_header == "100-continue" and httpversion >= (1, 1):
self.tcp_handler.wfile.write(
'HTTP/1.1 100 Continue\r\n'
'\r\n'
)
self.tcp_handler.wfile.flush()
del headers['expect']
if include_body:
content = self.read_http_body(
headers,
body_size_limit,
method,
None,
True
)
return http.Request(
form_in,
method,
scheme,
host,
port,
path,
httpversion,
headers,
content
)
def read_response(self, request_method, body_size_limit, include_body=True):
"""
Returns an http.Response
By default, both response header and body are read.
If include_body=False is specified, content may be one of the
following:
- None, if the response is technically allowed to have a response body
- "", if the response must not have a response body (e.g. it's a
response to a HEAD request)
"""
line = self.tcp_handler.rfile.readline()
# Possible leftover from previous message
if line == "\r\n" or line == "\n":
# Possible leftover from previous message
line = self.tcp_handler.rfile.readline()
return line
if not line:
raise HttpErrorConnClosed(502, "Server disconnect.")
parts = self.parse_response_line(line)
if not parts:
raise HttpError(502, "Invalid server response: %s" % repr(line))
proto, code, msg = parts
httpversion = self._parse_http_protocol(proto)
if httpversion is None:
raise HttpError(502, "Invalid HTTP version in line: %s" % repr(proto))
headers = self.read_headers()
if headers is None:
raise HttpError(502, "Invalid headers.")
if include_body:
content = self.read_http_body(
headers,
body_size_limit,
request_method,
code,
False
)
else:
# if include_body==False then a None content means the body should be
# read separately
content = None
return http.Response(httpversion, code, msg, headers, content)
def read_headers(self):
@ -56,177 +185,6 @@ class HTTP1Protocol(object):
return odict.ODictCaseless(ret)
def read_chunked(self, limit, is_request):
"""
Read a chunked HTTP body.
May raise HttpError.
"""
# FIXME: Should check if chunked is the final encoding in the headers
# http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-16#section-3.3
# 3.3 2.
total = 0
code = 400 if is_request else 502
while True:
line = self.tcp_handler.rfile.readline(128)
if line == "":
raise HttpErrorConnClosed(code, "Connection closed prematurely")
if line != '\r\n' and line != '\n':
try:
length = int(line, 16)
except ValueError:
raise HttpError(
code,
"Invalid chunked encoding length: %s" % line
)
total += length
if limit is not None and total > limit:
msg = "HTTP Body too large. Limit is %s," \
" chunked content longer than %s" % (limit, total)
raise HttpError(code, msg)
chunk = self.tcp_handler.rfile.read(length)
suffix = self.tcp_handler.rfile.readline(5)
if suffix != '\r\n':
raise HttpError(code, "Malformed chunked body")
yield line, chunk, '\r\n'
if length == 0:
return
@classmethod
def has_chunked_encoding(self, headers):
return "chunked" in [
i.lower() for i in http.get_header_tokens(headers, "transfer-encoding")
]
@classmethod
def parse_http_protocol(self, line):
"""
Parse an HTTP protocol declaration.
Returns a (major, minor) tuple, or None.
"""
if not line.startswith("HTTP/"):
return None
_, version = line.split('/', 1)
if "." not in version:
return None
major, minor = version.split('.', 1)
try:
major = int(major)
minor = int(minor)
except ValueError:
return None
return major, minor
@classmethod
def parse_init(self, line):
try:
method, url, protocol = string.split(line)
except ValueError:
return None
httpversion = self.parse_http_protocol(protocol)
if not httpversion:
return None
if not utils.isascii(method):
return None
return method, url, httpversion
@classmethod
def parse_init_connect(self, line):
"""
Returns (host, port, httpversion) if line is a valid CONNECT line.
http://tools.ietf.org/html/draft-luotonen-web-proxy-tunneling-01 section 3.1
"""
v = self.parse_init(line)
if not v:
return None
method, url, httpversion = v
if method.upper() != 'CONNECT':
return None
try:
host, port = url.split(":")
except ValueError:
return None
try:
port = int(port)
except ValueError:
return None
if not http.is_valid_port(port):
return None
if not http.is_valid_host(host):
return None
return host, port, httpversion
@classmethod
def parse_init_proxy(self, line):
v = self.parse_init(line)
if not v:
return None
method, url, httpversion = v
parts = http.parse_url(url)
if not parts:
return None
scheme, host, port, path = parts
return method, scheme, host, port, path, httpversion
@classmethod
def parse_init_http(self, line):
"""
Returns (method, url, httpversion)
"""
v = self.parse_init(line)
if not v:
return None
method, url, httpversion = v
if not utils.isascii(url):
return None
if not (url.startswith("/") or url == "*"):
return None
return method, url, httpversion
@classmethod
def connection_close(self, httpversion, headers):
"""
Checks the message to see if the client connection should be closed
according to RFC 2616 Section 8.1 Note that a connection should be
closed as well if the response has been read until end of the stream.
"""
# At first, check if we have an explicit Connection header.
if "connection" in headers:
toks = http.get_header_tokens(headers, "connection")
if "close" in toks:
return True
elif "keep-alive" in toks:
return False
# If we don't have a Connection header, HTTP 1.1 connections are assumed to
# be persistent
return httpversion != (1, 1)
@classmethod
def parse_response_line(self, line):
parts = line.strip().split(" ", 2)
if len(parts) == 2: # handle missing message gracefully
parts.append("")
if len(parts) != 3:
return None
proto, code, msg = parts
try:
code = int(code)
except ValueError:
return None
return (proto, code, msg)
def read_http_body(self, *args, **kwargs):
return "".join(
content for _, content, _ in self.read_http_body_chunked(*args, **kwargs)
@ -241,7 +199,7 @@ class HTTP1Protocol(object):
response_code,
is_request,
max_chunk_size=None
):
):
"""
Read an HTTP message body:
headers: An ODictCaseless object
@ -259,7 +217,7 @@ class HTTP1Protocol(object):
if expected_size is None:
if self.has_chunked_encoding(headers):
# Python 3: yield from
for x in self.read_chunked(limit, is_request):
for x in self._read_chunked(limit, is_request):
yield x
else: # pragma: nocover
raise HttpError(
@ -333,146 +291,6 @@ class HTTP1Protocol(object):
return -1
def read_request(self, include_body=True, body_size_limit=None, allow_empty=False):
"""
Parse an HTTP request from a file stream
Args:
include_body (bool): Read response body as well
body_size_limit (bool): Maximum body size
wfile (file): If specified, HTTP Expect headers are handled
automatically, by writing a HTTP 100 CONTINUE response to the stream.
Returns:
Request: The HTTP request
Raises:
HttpError: If the input is invalid.
"""
httpversion, host, port, scheme, method, path, headers, content = (
None, None, None, None, None, None, None, None)
request_line = self.get_request_line()
if not request_line:
if allow_empty:
return http.EmptyRequest()
else:
raise tcp.NetLibDisconnect()
request_line_parts = self.parse_init(request_line)
if not request_line_parts:
raise HttpError(
400,
"Bad HTTP request line: %s" % repr(request_line)
)
method, path, httpversion = request_line_parts
if path == '*' or path.startswith("/"):
form_in = "relative"
if not utils.isascii(path):
raise HttpError(
400,
"Bad HTTP request line: %s" % repr(request_line)
)
elif method.upper() == 'CONNECT':
form_in = "authority"
r = self.parse_init_connect(request_line)
if not r:
raise HttpError(
400,
"Bad HTTP request line: %s" % repr(request_line)
)
host, port, _ = r
return http.ConnectRequest(host, port)
else:
form_in = "absolute"
r = self.parse_init_proxy(request_line)
if not r:
raise HttpError(
400,
"Bad HTTP request line: %s" % repr(request_line)
)
_, scheme, host, port, path, _ = r
headers = self.read_headers()
if headers is None:
raise HttpError(400, "Invalid headers")
expect_header = headers.get_first("expect", "").lower()
if expect_header == "100-continue" and httpversion >= (1, 1):
self.tcp_handler.wfile.write(
'HTTP/1.1 100 Continue\r\n'
'\r\n'
)
self.tcp_handler.wfile.flush()
del headers['expect']
if include_body:
content = self.read_http_body(
headers,
body_size_limit,
method,
None,
True
)
return http.Request(
form_in,
method,
scheme,
host,
port,
path,
httpversion,
headers,
content
)
def read_response(self, request_method, body_size_limit, include_body=True):
"""
Returns an http.Response
By default, both response header and body are read.
If include_body=False is specified, content may be one of the
following:
- None, if the response is technically allowed to have a response body
- "", if the response must not have a response body (e.g. it's a
response to a HEAD request)
"""
line = self.tcp_handler.rfile.readline()
# Possible leftover from previous message
if line == "\r\n" or line == "\n":
line = self.tcp_handler.rfile.readline()
if not line:
raise HttpErrorConnClosed(502, "Server disconnect.")
parts = self.parse_response_line(line)
if not parts:
raise HttpError(502, "Invalid server response: %s" % repr(line))
proto, code, msg = parts
httpversion = self.parse_http_protocol(proto)
if httpversion is None:
raise HttpError(502, "Invalid HTTP version in line: %s" % repr(proto))
headers = self.read_headers()
if headers is None:
raise HttpError(502, "Invalid headers.")
if include_body:
content = self.read_http_body(
headers,
body_size_limit,
request_method,
code,
False
)
else:
# if include_body==False then a None content means the body should be
# read separately
content = None
return http.Response(httpversion, code, msg, headers, content)
@classmethod
def request_preamble(self, method, resource, http_major="1", http_minor="1"):
return '%s %s HTTP/%s.%s' % (
@ -485,3 +303,186 @@ class HTTP1Protocol(object):
if message is None:
message = status_codes.RESPONSES.get(code)
return 'HTTP/%s.%s %s %s' % (http_major, http_minor, code, message)
@classmethod
def has_chunked_encoding(self, headers):
return "chunked" in [
i.lower() for i in http.get_header_tokens(headers, "transfer-encoding")
]
def _get_request_line(self):
"""
Get a line, possibly preceded by a blank.
"""
line = self.tcp_handler.rfile.readline()
if line == "\r\n" or line == "\n":
# Possible leftover from previous message
line = self.tcp_handler.rfile.readline()
return line
def _read_chunked(self, limit, is_request):
"""
Read a chunked HTTP body.
May raise HttpError.
"""
# FIXME: Should check if chunked is the final encoding in the headers
# http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-16#section-3.3
# 3.3 2.
total = 0
code = 400 if is_request else 502
while True:
line = self.tcp_handler.rfile.readline(128)
if line == "":
raise HttpErrorConnClosed(code, "Connection closed prematurely")
if line != '\r\n' and line != '\n':
try:
length = int(line, 16)
except ValueError:
raise HttpError(
code,
"Invalid chunked encoding length: %s" % line
)
total += length
if limit is not None and total > limit:
msg = "HTTP Body too large. Limit is %s," \
" chunked content longer than %s" % (limit, total)
raise HttpError(code, msg)
chunk = self.tcp_handler.rfile.read(length)
suffix = self.tcp_handler.rfile.readline(5)
if suffix != '\r\n':
raise HttpError(code, "Malformed chunked body")
yield line, chunk, '\r\n'
if length == 0:
return
@classmethod
def _parse_http_protocol(self, line):
"""
Parse an HTTP protocol declaration.
Returns a (major, minor) tuple, or None.
"""
if not line.startswith("HTTP/"):
return None
_, version = line.split('/', 1)
if "." not in version:
return None
major, minor = version.split('.', 1)
try:
major = int(major)
minor = int(minor)
except ValueError:
return None
return major, minor
@classmethod
def _parse_init(self, line):
try:
method, url, protocol = string.split(line)
except ValueError:
return None
httpversion = self._parse_http_protocol(protocol)
if not httpversion:
return None
if not utils.isascii(method):
return None
return method, url, httpversion
@classmethod
def _parse_init_connect(self, line):
"""
Returns (host, port, httpversion) if line is a valid CONNECT line.
http://tools.ietf.org/html/draft-luotonen-web-proxy-tunneling-01 section 3.1
"""
v = self._parse_init(line)
if not v:
return None
method, url, httpversion = v
if method.upper() != 'CONNECT':
return None
try:
host, port = url.split(":")
except ValueError:
return None
try:
port = int(port)
except ValueError:
return None
if not http.is_valid_port(port):
return None
if not http.is_valid_host(host):
return None
return host, port, httpversion
@classmethod
def _parse_init_proxy(self, line):
v = self._parse_init(line)
if not v:
return None
method, url, httpversion = v
parts = http.parse_url(url)
if not parts:
return None
scheme, host, port, path = parts
return method, scheme, host, port, path, httpversion
@classmethod
def _parse_init_http(self, line):
"""
Returns (method, url, httpversion)
"""
v = self._parse_init(line)
if not v:
return None
method, url, httpversion = v
if not utils.isascii(url):
return None
if not (url.startswith("/") or url == "*"):
return None
return method, url, httpversion
@classmethod
def connection_close(self, httpversion, headers):
"""
Checks the message to see if the client connection should be closed
according to RFC 2616 Section 8.1 Note that a connection should be
closed as well if the response has been read until end of the stream.
"""
# At first, check if we have an explicit Connection header.
if "connection" in headers:
toks = http.get_header_tokens(headers, "connection")
if "close" in toks:
return True
elif "keep-alive" in toks:
return False
# If we don't have a Connection header, HTTP 1.1 connections are assumed to
# be persistent
return httpversion != (1, 1)
@classmethod
def parse_response_line(self, line):
parts = line.strip().split(" ", 2)
if len(parts) == 2: # handle missing message gracefully
parts.append("")
if len(parts) != 3:
return None
proto, code, msg = parts
try:
code = int(code)
except ValueError:
return None
return (proto, code, msg)

View File

@ -181,29 +181,29 @@ def test_expected_http_body_size():
def test_parse_http_protocol():
assert HTTP1Protocol.parse_http_protocol("HTTP/1.1") == (1, 1)
assert HTTP1Protocol.parse_http_protocol("HTTP/0.0") == (0, 0)
assert not HTTP1Protocol.parse_http_protocol("HTTP/a.1")
assert not HTTP1Protocol.parse_http_protocol("HTTP/1.a")
assert not HTTP1Protocol.parse_http_protocol("foo/0.0")
assert not HTTP1Protocol.parse_http_protocol("HTTP/x")
assert HTTP1Protocol._parse_http_protocol("HTTP/1.1") == (1, 1)
assert HTTP1Protocol._parse_http_protocol("HTTP/0.0") == (0, 0)
assert not HTTP1Protocol._parse_http_protocol("HTTP/a.1")
assert not HTTP1Protocol._parse_http_protocol("HTTP/1.a")
assert not HTTP1Protocol._parse_http_protocol("foo/0.0")
assert not HTTP1Protocol._parse_http_protocol("HTTP/x")
def test_parse_init_connect():
assert HTTP1Protocol.parse_init_connect("CONNECT host.com:443 HTTP/1.0")
assert not HTTP1Protocol.parse_init_connect("C\xfeONNECT host.com:443 HTTP/1.0")
assert not HTTP1Protocol.parse_init_connect("CONNECT \0host.com:443 HTTP/1.0")
assert not HTTP1Protocol.parse_init_connect("CONNECT host.com:444444 HTTP/1.0")
assert not HTTP1Protocol.parse_init_connect("bogus")
assert not HTTP1Protocol.parse_init_connect("GET host.com:443 HTTP/1.0")
assert not HTTP1Protocol.parse_init_connect("CONNECT host.com443 HTTP/1.0")
assert not HTTP1Protocol.parse_init_connect("CONNECT host.com:443 foo/1.0")
assert not HTTP1Protocol.parse_init_connect("CONNECT host.com:foo HTTP/1.0")
assert HTTP1Protocol._parse_init_connect("CONNECT host.com:443 HTTP/1.0")
assert not HTTP1Protocol._parse_init_connect("C\xfeONNECT host.com:443 HTTP/1.0")
assert not HTTP1Protocol._parse_init_connect("CONNECT \0host.com:443 HTTP/1.0")
assert not HTTP1Protocol._parse_init_connect("CONNECT host.com:444444 HTTP/1.0")
assert not HTTP1Protocol._parse_init_connect("bogus")
assert not HTTP1Protocol._parse_init_connect("GET host.com:443 HTTP/1.0")
assert not HTTP1Protocol._parse_init_connect("CONNECT host.com443 HTTP/1.0")
assert not HTTP1Protocol._parse_init_connect("CONNECT host.com:443 foo/1.0")
assert not HTTP1Protocol._parse_init_connect("CONNECT host.com:foo HTTP/1.0")
def test_parse_init_proxy():
u = "GET http://foo.com:8888/test HTTP/1.1"
m, s, h, po, pa, httpversion = HTTP1Protocol.parse_init_proxy(u)
m, s, h, po, pa, httpversion = HTTP1Protocol._parse_init_proxy(u)
assert m == "GET"
assert s == "http"
assert h == "foo.com"
@ -212,27 +212,27 @@ def test_parse_init_proxy():
assert httpversion == (1, 1)
u = "G\xfeET http://foo.com:8888/test HTTP/1.1"
assert not HTTP1Protocol.parse_init_proxy(u)
assert not HTTP1Protocol._parse_init_proxy(u)
assert not HTTP1Protocol.parse_init_proxy("invalid")
assert not HTTP1Protocol.parse_init_proxy("GET invalid HTTP/1.1")
assert not HTTP1Protocol.parse_init_proxy("GET http://foo.com:8888/test foo/1.1")
assert not HTTP1Protocol._parse_init_proxy("invalid")
assert not HTTP1Protocol._parse_init_proxy("GET invalid HTTP/1.1")
assert not HTTP1Protocol._parse_init_proxy("GET http://foo.com:8888/test foo/1.1")
def test_parse_init_http():
u = "GET /test HTTP/1.1"
m, u, httpversion = HTTP1Protocol.parse_init_http(u)
m, u, httpversion = HTTP1Protocol._parse_init_http(u)
assert m == "GET"
assert u == "/test"
assert httpversion == (1, 1)
u = "G\xfeET /test HTTP/1.1"
assert not HTTP1Protocol.parse_init_http(u)
assert not HTTP1Protocol._parse_init_http(u)
assert not HTTP1Protocol.parse_init_http("invalid")
assert not HTTP1Protocol.parse_init_http("GET invalid HTTP/1.1")
assert not HTTP1Protocol.parse_init_http("GET /test foo/1.1")
assert not HTTP1Protocol.parse_init_http("GET /test\xc0 HTTP/1.1")
assert not HTTP1Protocol._parse_init_http("invalid")
assert not HTTP1Protocol._parse_init_http("GET invalid HTTP/1.1")
assert not HTTP1Protocol._parse_init_http("GET /test foo/1.1")
assert not HTTP1Protocol._parse_init_http("GET /test\xc0 HTTP/1.1")
class TestReadHeaders:
@ -367,8 +367,8 @@ def test_read_response():
def test_get_request_line():
data = "\nfoo"
p = mock_protocol(data)
assert p.get_request_line() == "foo"
assert not p.get_request_line()
assert p._get_request_line() == "foo"
assert not p._get_request_line()
class TestReadRequest():