Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-23 00:01:36 +00:00
Code Issues Projects Releases Wiki Activity

Legibility

Browse Source
This commit is contained in:
Aldo Cortesi 2014-10-26 18:32:45 +13:00
parent 16654ad6a4
commit 340d0570bf
3 changed files with 128 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
libmproxy/protocol/http.py
View File

@ -27,7 +27,8 @@ def get_line(fp):
Get a line, possibly preceded by a blank.
"""
line = fp.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
if line == "\r\n" or line == "\n":
# Possible leftover from previous message
line = fp.readline()
if line == "":
raise tcp.NetLibDisconnect()
@ -241,25 +242,47 @@ class HTTPRequest(HTTPMessage):
is content associated, but not present. CONTENT_MISSING evaluates
to False to make checking for the presence of content natural.
form_in: The request form which mitmproxy has received. The following values are possible:
- relative (GET /index.html, OPTIONS *) (covers origin form and asterisk form)
- absolute (GET http://example.com:80/index.html)
- authority-form (CONNECT example.com:443)
Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3
form_in: The request form which mitmproxy has received. The following
values are possible:
form_out: The request form which mitmproxy has send out to the destination
- relative (GET /index.html, OPTIONS *) (covers origin form and
asterisk form)
- absolute (GET http://example.com:80/index.html)
- authority-form (CONNECT example.com:443)
Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3
form_out: The request form which mitmproxy has send out to the
destination
timestamp_start: Timestamp indicating when request transmission started
timestamp_end: Timestamp indicating when request transmission ended
"""
def __init__(self, form_in, method, scheme, host, port, path, httpversion, headers,
content, timestamp_start=None, timestamp_end=None, form_out=None):
def __init__(
self,
form_in,
method,
scheme,
host,
port,
path,
httpversion,
headers,
content,
timestamp_start=None,
timestamp_end=None,
form_out=None
):
assert isinstance(headers, ODictCaseless) or not headers
HTTPMessage.__init__(self, httpversion, headers, content, timestamp_start,
timestamp_end)
HTTPMessage.__init__(
self,
httpversion,
headers,
content,
timestamp_start,
timestamp_end
)
self.form_in = form_in
self.method = method
self.scheme = scheme
@ -312,7 +335,8 @@ class HTTPRequest(HTTPMessage):
request_line = get_line(rfile)
if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start
if hasattr(rfile, "first_byte_timestamp"):
# more accurate timestamp_start
timestamp_start = rfile.first_byte_timestamp
request_line_parts = http.parse_init(request_line)
@ -683,7 +707,9 @@ class HTTPResponse(HTTPMessage):
return "<HTTPResponse: {code} {msg} ({contenttype}, {size})>".format(
code=self.code,
msg=self.msg,
contenttype=self.headers.get_first("content-type", "unknown content type"),
contenttype=self.headers.get_first(
"content-type", "unknown content type"
),
size=size
)
@ -704,7 +730,8 @@ class HTTPResponse(HTTPMessage):
body_size_limit,
include_body=include_body)
if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start
if hasattr(rfile, "first_byte_timestamp"):
# more accurate timestamp_start
timestamp_start = rfile.first_byte_timestamp
if include_body:
@ -745,7 +772,11 @@ class HTTPResponse(HTTPMessage):
def _assemble_head(self, preserve_transfer_encoding=False):
return '%s\r\n%s\r\n' % (
self._assemble_first_line(), self._assemble_headers(preserve_transfer_encoding=preserve_transfer_encoding))
self._assemble_first_line(),
self._assemble_headers(
preserve_transfer_encoding=preserve_transfer_encoding
)
)
def assemble(self):
"""
@ -755,7 +786,10 @@ class HTTPResponse(HTTPMessage):
Raises an Exception if the request cannot be assembled.
"""
if self.content == CONTENT_MISSING:
raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING")
raise proxy.ProxyError(
502,
"Cannot assemble flow with CONTENT_MISSING"
)
head = self._assemble_head()
if self.content:
return head + self.content
@ -822,8 +856,9 @@ class HTTPResponse(HTTPMessage):
pairs = [pair.partition("=") for pair in header.split(';')]
cookie_name = pairs[0][0] # the key of the first key/value pairs
cookie_value = pairs[0][2] # the value of the first key/value pairs
cookie_parameters = {key.strip().lower(): value.strip() for key, sep, value in
pairs[1:]}
cookie_parameters = {
key.strip().lower(): value.strip() for key, sep, value in pairs[1:]
}
cookies.append((cookie_name, (cookie_value, cookie_parameters)))
return dict(cookies)
@ -856,7 +891,8 @@ class HTTPFlow(Flow):
self.response = None
"""@type: HTTPResponse"""
self.intercepting = False # FIXME: Should that rather be an attribute of Flow?
# FIXME: Should that rather be an attribute of Flow?
self.intercepting = False
_stateobject_attributes = Flow._stateobject_attributes.copy()
_stateobject_attributes.update(
@ -944,7 +980,9 @@ class HTTPFlow(Flow):
class HttpAuthenticationError(Exception):
def __init__(self, auth_headers=None):
super(HttpAuthenticationError, self).__init__("Proxy Authentication Required")
super(HttpAuthenticationError, self).__init__(
"Proxy Authentication Required"
)
self.headers = auth_headers
self.code = 407
@ -1114,7 +1152,12 @@ class HTTPHandler(ProtocolHandler):
def handle_server_reconnect(self, state):
if state["state"] == "connect":
send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False)
send_connect_request(
self.c.server_conn,
state["host"],
state["port"],
update_state=False
)
else: # pragma: nocover
raise RuntimeError("Unknown State: %s" % state["state"])
@ -1138,11 +1181,11 @@ class HTTPHandler(ProtocolHandler):
self.c.log(message_debug, level="debug")
if flow:
# TODO: no flows without request or with both request and response at the moment.
# TODO: no flows without request or with both request and response
# at the moment.
if flow.request and not flow.response:
flow.error = Error(message or message_debug)
self.c.channel.ask("error", flow)
try:
code = getattr(error, "code", 502)
headers = getattr(error, "headers", None)
@ -1156,12 +1199,22 @@ class HTTPHandler(ProtocolHandler):
def send_error(self, code, message, headers):
response = http_status.RESPONSES.get(code, "Unknown")
html_content = '<html><head>\n<title>%d %s</title>\n</head>\n<body>\n%s\n</body>\n</html>' % \
(code, response, message)
html_content = """
<html>
<head>
<title>%d %s</title>
</head>
<body %s</body>
</html>
""" % (code, response, message)
self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response))
self.c.client_conn.wfile.write("Server: %s\r\n" % self.c.config.server_version)
self.c.client_conn.wfile.write(
"Server: %s\r\n" % self.c.config.server_version
)
self.c.client_conn.wfile.write("Content-type: text/html\r\n")
self.c.client_conn.wfile.write("Content-Length: %d\r\n" % len(html_content))
self.c.client_conn.wfile.write(
"Content-Length: %d\r\n" % len(html_content)
)
if headers:
for key, value in headers.items():
self.c.client_conn.wfile.write("%s: %s\r\n" % (key, value))
@ -1201,11 +1254,15 @@ class HTTPHandler(ProtocolHandler):
# Now we can process the request.
if request.form_in == "authority":
if self.c.client_conn.ssl_established:
raise http.HttpError(400, "Must not CONNECT on already encrypted connection")
raise http.HttpError(
400,
"Must not CONNECT on already encrypted connection"
)
if self.c.config.mode == "regular":
self.c.set_server_address((request.host, request.port))
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow
# Update server_conn attribute on the flow
flow.server_conn = self.c.server_conn
self.c.establish_server_connection()
self.c.client_conn.send(
'HTTP/1.1 200 Connection established\r\n' +
@ -1217,7 +1274,9 @@ class HTTPHandler(ProtocolHandler):
elif self.c.config.mode == "upstream":
return None
else:
pass # CONNECT should never occur if we don't expect absolute-form requests
# CONNECT should never occur if we don't expect absolute-form
# requests
pass
elif request.form_in == self.expected_form_in:
@ -1225,61 +1284,77 @@ class HTTPHandler(ProtocolHandler):
if request.form_in == "absolute":
if request.scheme != "http":
raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme)
raise http.HttpError(
400,
"Invalid request scheme: %s" % request.scheme
)
if self.c.config.mode == "regular":
# Update info so that an inline script sees the correct value at flow.server_conn
# Update info so that an inline script sees the correct
# value at flow.server_conn
self.c.set_server_address((request.host, request.port))
flow.server_conn = self.c.server_conn
return None
raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" %
(self.expected_form_in, request.form_in))
raise http.HttpError(
400, "Invalid HTTP request form (expected: %s, got: %s)" % (
self.expected_form_in, request.form_in
)
)
def process_server_address(self, flow):
# Depending on the proxy mode, server handling is entirely different
# We provide a mostly unified API to the user, which needs to be unfiddled here
# We provide a mostly unified API to the user, which needs to be
# unfiddled here
# ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 )
address = netlib.tcp.Address((flow.request.host, flow.request.port))
ssl = (flow.request.scheme == "https")
if self.c.config.mode == "upstream":
# The connection to the upstream proxy may have a state we may need to take into account.
# The connection to the upstream proxy may have a state we may need
# to take into account.
connected_to = None
for s in flow.server_conn.state:
if s[0] == "http" and s[1]["state"] == "connect":
connected_to = tcp.Address((s[1]["host"], s[1]["port"]))
# We need to reconnect if the current flow either requires a (possibly impossible)
# change to the connection state, e.g. the host has changed but we already CONNECTed somewhere else.
# We need to reconnect if the current flow either requires a
# (possibly impossible) change to the connection state, e.g. the
# host has changed but we already CONNECTed somewhere else.
needs_server_change = (
ssl != self.c.server_conn.ssl_established
or
(connected_to and address != connected_to) # HTTP proxying is "stateless", CONNECT isn't.
# HTTP proxying is "stateless", CONNECT isn't.
(connected_to and address != connected_to)
)
if needs_server_change:
# force create new connection to the proxy server to reset state
self.live.change_server(self.c.server_conn.address, force=True)
if ssl:
send_connect_request(self.c.server_conn, address.host, address.port)
send_connect_request(
self.c.server_conn,
address.host,
address.port
)
self.c.establish_ssl(server=True)
else:
# If we're not in upstream mode, we just want to update the host and possibly establish TLS.
self.live.change_server(address, ssl=ssl) # this is a no op if the addresses match.
# If we're not in upstream mode, we just want to update the host and
# possibly establish TLS. This is a no op if the addresses match.
self.live.change_server(address, ssl=ssl)
flow.server_conn = self.c.server_conn
def send_response_to_client(self, flow):
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can send it to the client straight away.
# we already received the full response from the server and can send
# it to the client straight away.
self.c.client_conn.send(flow.response.assemble())
else:
# streaming:
# First send the headers and then transfer the response incrementally:
# First send the headers and then transfer the response
# incrementally:
h = flow.response._assemble_head(preserve_transfer_encoding=True)
self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
@ -1293,7 +1368,8 @@ class HTTPHandler(ProtocolHandler):
def check_close_connection(self, flow):
"""
Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so.
Checks if the connection should be closed depending on the HTTP
semantics. Returns True, if so.
"""
close_connection = (
http.connection_close(flow.request.httpversion, flow.request.headers) or

4
test/fuzzing/straight_stream
View File

@ -1,6 +1,6 @@
mitmdump: $MITMDUMP -q --stream 1
pathod: $PATHOD -q
#pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns
pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 /tmp/err
pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns
#pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 /tmp/err

2
test/fuzzing/straight_stream_patterns
View File

@ -3,3 +3,5 @@ get:'http://localhost:9999/p/':s'200:b"foo"':ir,'a'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'9'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,':'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'"'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'-'
get:'http://localhost:9999/p/':s'200:b"foo"':dr