mirror of
https://github.com/Grasscutters/mitmproxy.git
synced 2024-11-24 00:31:33 +00:00
467 lines
14 KiB
Python
467 lines
14 KiB
Python
from io import BytesIO
|
|
import textwrap
|
|
from http.http1.protocol import _parse_authority_form
|
|
from netlib.exceptions import HttpSyntaxException, HttpReadDisconnect, HttpException
|
|
|
|
from netlib import http, tcp, tutils
|
|
from netlib.http import semantics, Headers
|
|
from netlib.http.http1 import HTTP1Protocol, read_message_body, read_request, \
|
|
read_message_body_chunked, expected_http_body_size
|
|
from ... import tservers
|
|
|
|
|
|
class NoContentLengthHTTPHandler(tcp.BaseHandler):
|
|
def handle(self):
|
|
self.wfile.write("HTTP/1.1 200 OK\r\n\r\nbar\r\n\r\n")
|
|
self.wfile.flush()
|
|
|
|
|
|
def mock_protocol(data=''):
|
|
rfile = BytesIO(data)
|
|
wfile = BytesIO()
|
|
return HTTP1Protocol(rfile=rfile, wfile=wfile)
|
|
|
|
|
|
def match_http_string(data):
|
|
return textwrap.dedent(data).strip().replace('\n', '\r\n')
|
|
|
|
|
|
def test_stripped_chunked_encoding_no_content():
|
|
"""
|
|
https://github.com/mitmproxy/mitmproxy/issues/186
|
|
"""
|
|
|
|
r = tutils.treq(content="")
|
|
r.headers["Transfer-Encoding"] = "chunked"
|
|
assert "Content-Length" in mock_protocol()._assemble_request_headers(r)
|
|
|
|
r = tutils.tresp(content="")
|
|
r.headers["Transfer-Encoding"] = "chunked"
|
|
assert "Content-Length" in mock_protocol()._assemble_response_headers(r)
|
|
|
|
|
|
def test_read_chunked():
|
|
req = tutils.treq(None)
|
|
req.headers["Transfer-Encoding"] = "chunked"
|
|
|
|
data = b"1\r\na\r\n0\r\n"
|
|
with tutils.raises(HttpSyntaxException):
|
|
read_message_body(BytesIO(data), req)
|
|
|
|
data = b"1\r\na\r\n0\r\n\r\n"
|
|
assert read_message_body(BytesIO(data), req) == b"a"
|
|
|
|
data = b"\r\n\r\n1\r\na\r\n1\r\nb\r\n0\r\n\r\n"
|
|
assert read_message_body(BytesIO(data), req) == b"ab"
|
|
|
|
data = b"\r\n"
|
|
with tutils.raises("closed prematurely"):
|
|
read_message_body(BytesIO(data), req)
|
|
|
|
data = b"1\r\nfoo"
|
|
with tutils.raises("malformed chunked body"):
|
|
read_message_body(BytesIO(data), req)
|
|
|
|
data = b"foo\r\nfoo"
|
|
with tutils.raises(HttpSyntaxException):
|
|
read_message_body(BytesIO(data), req)
|
|
|
|
data = b"5\r\naaaaa\r\n0\r\n\r\n"
|
|
with tutils.raises("too large"):
|
|
read_message_body(BytesIO(data), req, limit=2)
|
|
|
|
|
|
def test_connection_close():
|
|
headers = Headers()
|
|
assert HTTP1Protocol.connection_close((1, 0), headers)
|
|
assert not HTTP1Protocol.connection_close((1, 1), headers)
|
|
|
|
headers["connection"] = "keep-alive"
|
|
assert not HTTP1Protocol.connection_close((1, 1), headers)
|
|
|
|
headers["connection"] = "close"
|
|
assert HTTP1Protocol.connection_close((1, 1), headers)
|
|
|
|
|
|
def test_read_http_body_request():
|
|
headers = Headers()
|
|
data = "testing"
|
|
assert mock_protocol(data).read_http_body(headers, None, "GET", None, True) == ""
|
|
|
|
|
|
def test_read_http_body_response():
|
|
headers = Headers()
|
|
data = "testing"
|
|
assert mock_protocol(data).read_http_body(headers, None, "GET", 200, False) == "testing"
|
|
|
|
|
|
def test_read_http_body():
|
|
# test default case
|
|
headers = Headers()
|
|
headers["content-length"] = "7"
|
|
data = "testing"
|
|
assert mock_protocol(data).read_http_body(headers, None, "GET", 200, False) == "testing"
|
|
|
|
# test content length: invalid header
|
|
headers["content-length"] = "foo"
|
|
data = "testing"
|
|
tutils.raises(
|
|
http.HttpError,
|
|
mock_protocol(data).read_http_body,
|
|
headers, None, "GET", 200, False
|
|
)
|
|
|
|
# test content length: invalid header #2
|
|
headers["content-length"] = "-1"
|
|
data = "testing"
|
|
tutils.raises(
|
|
http.HttpError,
|
|
mock_protocol(data).read_http_body,
|
|
headers, None, "GET", 200, False
|
|
)
|
|
|
|
# test content length: content length > actual content
|
|
headers["content-length"] = "5"
|
|
data = "testing"
|
|
tutils.raises(
|
|
http.HttpError,
|
|
mock_protocol(data).read_http_body,
|
|
headers, 4, "GET", 200, False
|
|
)
|
|
|
|
# test content length: content length < actual content
|
|
data = "testing"
|
|
assert len(mock_protocol(data).read_http_body(headers, None, "GET", 200, False)) == 5
|
|
|
|
# test no content length: limit > actual content
|
|
headers = Headers()
|
|
data = "testing"
|
|
assert len(mock_protocol(data).read_http_body(headers, 100, "GET", 200, False)) == 7
|
|
|
|
# test no content length: limit < actual content
|
|
data = "testing"
|
|
tutils.raises(
|
|
http.HttpError,
|
|
mock_protocol(data).read_http_body,
|
|
headers, 4, "GET", 200, False
|
|
)
|
|
|
|
# test chunked
|
|
headers = Headers()
|
|
headers["transfer-encoding"] = "chunked"
|
|
data = "5\r\naaaaa\r\n0\r\n\r\n"
|
|
assert mock_protocol(data).read_http_body(headers, 100, "GET", 200, False) == "aaaaa"
|
|
|
|
|
|
def test_expected_http_body_size():
|
|
# gibber in the content-length field
|
|
headers = Headers(content_length="foo")
|
|
with tutils.raises(HttpSyntaxException):
|
|
expected_http_body_size(headers, False, "GET", 200) is None
|
|
# negative number in the content-length field
|
|
headers = Headers(content_length="-7")
|
|
with tutils.raises(HttpSyntaxException):
|
|
expected_http_body_size(headers, False, "GET", 200) is None
|
|
# explicit length
|
|
headers = Headers(content_length="5")
|
|
assert expected_http_body_size(headers, False, "GET", 200) == 5
|
|
# no length
|
|
headers = Headers()
|
|
assert expected_http_body_size(headers, False, "GET", 200) == -1
|
|
# no length request
|
|
headers = Headers()
|
|
assert expected_http_body_size(headers, True, "GET", None) == 0
|
|
# expect header
|
|
headers = Headers(content_length="5", expect="100-continue")
|
|
assert expected_http_body_size(headers, True, "GET", None) == 0
|
|
|
|
|
|
def test_parse_init_connect():
|
|
assert _parse_authority_form(b"CONNECT host.com:443 HTTP/1.0")
|
|
tutils.raises(ValueError,_parse_authority_form, b"\0host.com:443")
|
|
tutils.raises(ValueError,_parse_authority_form, b"host.com:444444")
|
|
tutils.raises(ValueError,_parse_authority_form, b"CONNECT host.com443 HTTP/1.0")
|
|
tutils.raises(ValueError,_parse_authority_form, b"CONNECT host.com:foo HTTP/1.0")
|
|
|
|
|
|
def test_parse_init_proxy():
|
|
u = b"GET http://foo.com:8888/test HTTP/1.1"
|
|
m, s, h, po, pa, httpversion = HTTP1Protocol._parse_absolute_form(u)
|
|
assert m == "GET"
|
|
assert s == "http"
|
|
assert h == "foo.com"
|
|
assert po == 8888
|
|
assert pa == "/test"
|
|
assert httpversion == (1, 1)
|
|
|
|
u = "G\xfeET http://foo.com:8888/test HTTP/1.1"
|
|
assert not HTTP1Protocol._parse_absolute_form(u)
|
|
|
|
with tutils.raises(ValueError):
|
|
assert not HTTP1Protocol._parse_absolute_form("invalid")
|
|
with tutils.raises(ValueError):
|
|
assert not HTTP1Protocol._parse_absolute_form("GET invalid HTTP/1.1")
|
|
with tutils.raises(ValueError):
|
|
assert not HTTP1Protocol._parse_absolute_form("GET http://foo.com:8888/test foo/1.1")
|
|
|
|
|
|
def test_parse_init_http():
|
|
u = "GET /test HTTP/1.1"
|
|
m, u, httpversion = HTTP1Protocol._parse_init_http(u)
|
|
assert m == "GET"
|
|
assert u == "/test"
|
|
assert httpversion == (1, 1)
|
|
|
|
u = "G\xfeET /test HTTP/1.1"
|
|
assert not HTTP1Protocol._parse_init_http(u)
|
|
|
|
assert not HTTP1Protocol._parse_init_http("invalid")
|
|
assert not HTTP1Protocol._parse_init_http("GET invalid HTTP/1.1")
|
|
assert not HTTP1Protocol._parse_init_http("GET /test foo/1.1")
|
|
assert not HTTP1Protocol._parse_init_http("GET /test\xc0 HTTP/1.1")
|
|
|
|
|
|
class TestReadHeaders:
|
|
|
|
def _read(self, data, verbatim=False):
|
|
if not verbatim:
|
|
data = textwrap.dedent(data)
|
|
data = data.strip()
|
|
return mock_protocol(data).read_headers()
|
|
|
|
def test_read_simple(self):
|
|
data = """
|
|
Header: one
|
|
Header2: two
|
|
\r\n
|
|
"""
|
|
headers = self._read(data)
|
|
assert headers.fields == [["Header", "one"], ["Header2", "two"]]
|
|
|
|
def test_read_multi(self):
|
|
data = """
|
|
Header: one
|
|
Header: two
|
|
\r\n
|
|
"""
|
|
headers = self._read(data)
|
|
assert headers.fields == [["Header", "one"], ["Header", "two"]]
|
|
|
|
def test_read_continued(self):
|
|
data = """
|
|
Header: one
|
|
\ttwo
|
|
Header2: three
|
|
\r\n
|
|
"""
|
|
headers = self._read(data)
|
|
assert headers.fields == [["Header", "one\r\n two"], ["Header2", "three"]]
|
|
|
|
def test_read_continued_err(self):
|
|
data = "\tfoo: bar\r\n"
|
|
assert self._read(data, True) is None
|
|
|
|
def test_read_err(self):
|
|
data = """
|
|
foo
|
|
"""
|
|
assert self._read(data) is None
|
|
|
|
|
|
class TestReadRequest(object):
|
|
|
|
def tst(self, data, **kwargs):
|
|
return mock_protocol(data).read_request(**kwargs)
|
|
|
|
def test_invalid(self):
|
|
tutils.raises(
|
|
"bad http request",
|
|
self.tst,
|
|
"xxx"
|
|
)
|
|
tutils.raises(
|
|
"bad http request line",
|
|
self.tst,
|
|
"get /\xff HTTP/1.1"
|
|
)
|
|
tutils.raises(
|
|
"invalid headers",
|
|
self.tst,
|
|
"get / HTTP/1.1\r\nfoo"
|
|
)
|
|
tutils.raises(
|
|
HttpReadDisconnect,
|
|
self.tst,
|
|
"\r\n"
|
|
)
|
|
|
|
def test_asterisk_form_in(self):
|
|
v = self.tst("OPTIONS * HTTP/1.1")
|
|
assert v.form_in == "relative"
|
|
assert v.method == "OPTIONS"
|
|
|
|
def test_absolute_form_in(self):
|
|
tutils.raises(
|
|
"Bad HTTP request line",
|
|
self.tst,
|
|
"GET oops-no-protocol.com HTTP/1.1"
|
|
)
|
|
v = self.tst("GET http://address:22/ HTTP/1.1")
|
|
assert v.form_in == "absolute"
|
|
assert v.port == 22
|
|
assert v.host == "address"
|
|
assert v.scheme == "http"
|
|
|
|
def test_connect(self):
|
|
tutils.raises(
|
|
"Bad HTTP request line",
|
|
self.tst,
|
|
"CONNECT oops-no-port.com HTTP/1.1"
|
|
)
|
|
v = self.tst("CONNECT foo.com:443 HTTP/1.1")
|
|
assert v.form_in == "authority"
|
|
assert v.method == "CONNECT"
|
|
assert v.port == 443
|
|
assert v.host == "foo.com"
|
|
|
|
def test_expect(self):
|
|
data = (
|
|
b"GET / HTTP/1.1\r\n"
|
|
b"Content-Length: 3\r\n"
|
|
b"Expect: 100-continue\r\n"
|
|
b"\r\n"
|
|
b"foobar"
|
|
)
|
|
|
|
rfile = BytesIO(data)
|
|
r = read_request(rfile)
|
|
assert r.body == b""
|
|
assert rfile.read(-1) == b"foobar"
|
|
|
|
|
|
class TestReadResponse(object):
|
|
def tst(self, data, method, body_size_limit, include_body=True):
|
|
data = textwrap.dedent(data)
|
|
return mock_protocol(data).read_response(
|
|
method, body_size_limit, include_body=include_body
|
|
)
|
|
|
|
def test_errors(self):
|
|
tutils.raises("server disconnect", self.tst, "", "GET", None)
|
|
tutils.raises("invalid server response", self.tst, "foo", "GET", None)
|
|
|
|
def test_simple(self):
|
|
data = """
|
|
HTTP/1.1 200
|
|
"""
|
|
assert self.tst(data, "GET", None) == http.Response(
|
|
(1, 1), 200, '', Headers(), ''
|
|
)
|
|
|
|
def test_simple_message(self):
|
|
data = """
|
|
HTTP/1.1 200 OK
|
|
"""
|
|
assert self.tst(data, "GET", None) == http.Response(
|
|
(1, 1), 200, 'OK', Headers(), ''
|
|
)
|
|
|
|
def test_invalid_http_version(self):
|
|
data = """
|
|
HTTP/x 200 OK
|
|
"""
|
|
tutils.raises("invalid http version", self.tst, data, "GET", None)
|
|
|
|
def test_invalid_status_code(self):
|
|
data = """
|
|
HTTP/1.1 xx OK
|
|
"""
|
|
tutils.raises("invalid server response", self.tst, data, "GET", None)
|
|
|
|
def test_valid_with_continue(self):
|
|
data = """
|
|
HTTP/1.1 100 CONTINUE
|
|
|
|
HTTP/1.1 200 OK
|
|
"""
|
|
assert self.tst(data, "GET", None) == http.Response(
|
|
(1, 1), 100, 'CONTINUE', Headers(), ''
|
|
)
|
|
|
|
def test_simple_body(self):
|
|
data = """
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 3
|
|
|
|
foo
|
|
"""
|
|
assert self.tst(data, "GET", None).body == 'foo'
|
|
assert self.tst(data, "HEAD", None).body == ''
|
|
|
|
def test_invalid_headers(self):
|
|
data = """
|
|
HTTP/1.1 200 OK
|
|
\tContent-Length: 3
|
|
|
|
foo
|
|
"""
|
|
tutils.raises("invalid headers", self.tst, data, "GET", None)
|
|
|
|
def test_without_body(self):
|
|
data = """
|
|
HTTP/1.1 200 OK
|
|
Content-Length: 3
|
|
|
|
foo
|
|
"""
|
|
assert self.tst(data, "GET", None, include_body=False).body is None
|
|
|
|
|
|
class TestReadResponseNoContentLength(tservers.ServerTestBase):
|
|
handler = NoContentLengthHTTPHandler
|
|
|
|
def test_no_content_length(self):
|
|
c = tcp.TCPClient(("127.0.0.1", self.port))
|
|
c.connect()
|
|
resp = HTTP1Protocol(c).read_response("GET", None)
|
|
assert resp.body == "bar\r\n\r\n"
|
|
|
|
|
|
class TestAssembleRequest(object):
|
|
def test_simple(self):
|
|
req = tutils.treq()
|
|
b = HTTP1Protocol().assemble_request(req)
|
|
assert b == match_http_string("""
|
|
GET /path HTTP/1.1
|
|
header: qvalue
|
|
Host: address:22
|
|
Content-Length: 7
|
|
|
|
content""")
|
|
|
|
def test_body_missing(self):
|
|
req = tutils.treq(content=semantics.CONTENT_MISSING)
|
|
tutils.raises(http.HttpError, HTTP1Protocol().assemble_request, req)
|
|
|
|
def test_not_a_request(self):
|
|
tutils.raises(AssertionError, HTTP1Protocol().assemble_request, 'foo')
|
|
|
|
|
|
class TestAssembleResponse(object):
|
|
def test_simple(self):
|
|
resp = tutils.tresp()
|
|
b = HTTP1Protocol().assemble_response(resp)
|
|
assert b == match_http_string("""
|
|
HTTP/1.1 200 OK
|
|
header_response: svalue
|
|
Content-Length: 7
|
|
|
|
message""")
|
|
|
|
def test_body_missing(self):
|
|
resp = tutils.tresp(content=semantics.CONTENT_MISSING)
|
|
tutils.raises(http.HttpError, HTTP1Protocol().assemble_response, resp)
|
|
|
|
def test_not_a_request(self):
|
|
tutils.raises(AssertionError, HTTP1Protocol().assemble_response, 'foo')
|