Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-23 00:01:36 +00:00
Code Issues Projects Releases Wiki Activity

Legibility

Browse Source
This commit is contained in:
Aldo Cortesi 2014-10-26 18:32:45 +13:00
parent 16654ad6a4
commit 340d0570bf
3 changed files with 128 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
libmproxy/protocol/http.py
View File

@ -27,7 +27,8 @@ def get_line(fp):
Get a line, possibly preceded by a blank. Get a line, possibly preceded by a blank.
""" """
line = fp.readline() line = fp.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message if line == "\r\n" or line == "\n":
# Possible leftover from previous message
line = fp.readline() line = fp.readline()
if line == "": if line == "":
raise tcp.NetLibDisconnect() raise tcp.NetLibDisconnect()
@ -241,25 +242,47 @@ class HTTPRequest(HTTPMessage):
is content associated, but not present. CONTENT_MISSING evaluates is content associated, but not present. CONTENT_MISSING evaluates
to False to make checking for the presence of content natural. to False to make checking for the presence of content natural.
form_in: The request form which mitmproxy has received. The following values are possible: form_in: The request form which mitmproxy has received. The following
- relative (GET /index.html, OPTIONS *) (covers origin form and asterisk form) values are possible:
- absolute (GET http://example.com:80/index.html)
- authority-form (CONNECT example.com:443)
Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3
form_out: The request form which mitmproxy has send out to the destination - relative (GET /index.html, OPTIONS *) (covers origin form and
asterisk form)
- absolute (GET http://example.com:80/index.html)
- authority-form (CONNECT example.com:443)
Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3
form_out: The request form which mitmproxy has send out to the
destination
timestamp_start: Timestamp indicating when request transmission started timestamp_start: Timestamp indicating when request transmission started
timestamp_end: Timestamp indicating when request transmission ended timestamp_end: Timestamp indicating when request transmission ended
""" """
def __init__(self, form_in, method, scheme, host, port, path, httpversion, headers, def __init__(
content, timestamp_start=None, timestamp_end=None, form_out=None): self,
form_in,
method,
scheme,
host,
port,
path,
httpversion,
headers,
content,
timestamp_start=None,
timestamp_end=None,
form_out=None
):
assert isinstance(headers, ODictCaseless) or not headers assert isinstance(headers, ODictCaseless) or not headers
HTTPMessage.__init__(self, httpversion, headers, content, timestamp_start, HTTPMessage.__init__(
timestamp_end) self,
httpversion,
headers,
content,
timestamp_start,
timestamp_end
)
self.form_in = form_in self.form_in = form_in
self.method = method self.method = method
self.scheme = scheme self.scheme = scheme
@ -312,7 +335,8 @@ class HTTPRequest(HTTPMessage):
request_line = get_line(rfile) request_line = get_line(rfile)
if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start if hasattr(rfile, "first_byte_timestamp"):
# more accurate timestamp_start
timestamp_start = rfile.first_byte_timestamp timestamp_start = rfile.first_byte_timestamp
request_line_parts = http.parse_init(request_line) request_line_parts = http.parse_init(request_line)
@ -683,7 +707,9 @@ class HTTPResponse(HTTPMessage):
return "<HTTPResponse: {code} {msg} ({contenttype}, {size})>".format( return "<HTTPResponse: {code} {msg} ({contenttype}, {size})>".format(
code=self.code, code=self.code,
msg=self.msg, msg=self.msg,
contenttype=self.headers.get_first("content-type", "unknown content type"), contenttype=self.headers.get_first(
"content-type", "unknown content type"
),
size=size size=size
) )
@ -704,7 +730,8 @@ class HTTPResponse(HTTPMessage):
body_size_limit, body_size_limit,
include_body=include_body) include_body=include_body)
if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start if hasattr(rfile, "first_byte_timestamp"):
# more accurate timestamp_start
timestamp_start = rfile.first_byte_timestamp timestamp_start = rfile.first_byte_timestamp
if include_body: if include_body:
@ -745,7 +772,11 @@ class HTTPResponse(HTTPMessage):
def _assemble_head(self, preserve_transfer_encoding=False): def _assemble_head(self, preserve_transfer_encoding=False):
return '%s\r\n%s\r\n' % ( return '%s\r\n%s\r\n' % (
self._assemble_first_line(), self._assemble_headers(preserve_transfer_encoding=preserve_transfer_encoding)) self._assemble_first_line(),
self._assemble_headers(
preserve_transfer_encoding=preserve_transfer_encoding
)
)
def assemble(self): def assemble(self):
""" """
@ -755,7 +786,10 @@ class HTTPResponse(HTTPMessage):
Raises an Exception if the request cannot be assembled. Raises an Exception if the request cannot be assembled.
""" """
if self.content == CONTENT_MISSING: if self.content == CONTENT_MISSING:
raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING") raise proxy.ProxyError(
502,
"Cannot assemble flow with CONTENT_MISSING"
)
head = self._assemble_head() head = self._assemble_head()
if self.content: if self.content:
return head + self.content return head + self.content
@ -822,8 +856,9 @@ class HTTPResponse(HTTPMessage):
pairs = [pair.partition("=") for pair in header.split(';')] pairs = [pair.partition("=") for pair in header.split(';')]
cookie_name = pairs[0][0] # the key of the first key/value pairs cookie_name = pairs[0][0] # the key of the first key/value pairs
cookie_value = pairs[0][2] # the value of the first key/value pairs cookie_value = pairs[0][2] # the value of the first key/value pairs
cookie_parameters = {key.strip().lower(): value.strip() for key, sep, value in cookie_parameters = {
pairs[1:]} key.strip().lower(): value.strip() for key, sep, value in pairs[1:]
}
cookies.append((cookie_name, (cookie_value, cookie_parameters))) cookies.append((cookie_name, (cookie_value, cookie_parameters)))
return dict(cookies) return dict(cookies)
@ -856,7 +891,8 @@ class HTTPFlow(Flow):
self.response = None self.response = None
"""@type: HTTPResponse""" """@type: HTTPResponse"""
self.intercepting = False # FIXME: Should that rather be an attribute of Flow? # FIXME: Should that rather be an attribute of Flow?
self.intercepting = False
_stateobject_attributes = Flow._stateobject_attributes.copy() _stateobject_attributes = Flow._stateobject_attributes.copy()
_stateobject_attributes.update( _stateobject_attributes.update(
@ -944,7 +980,9 @@ class HTTPFlow(Flow):
class HttpAuthenticationError(Exception): class HttpAuthenticationError(Exception):
def __init__(self, auth_headers=None): def __init__(self, auth_headers=None):
super(HttpAuthenticationError, self).__init__("Proxy Authentication Required") super(HttpAuthenticationError, self).__init__(
"Proxy Authentication Required"
)
self.headers = auth_headers self.headers = auth_headers
self.code = 407 self.code = 407
@ -1114,7 +1152,12 @@ class HTTPHandler(ProtocolHandler):
def handle_server_reconnect(self, state): def handle_server_reconnect(self, state):
if state["state"] == "connect": if state["state"] == "connect":
send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False) send_connect_request(
self.c.server_conn,
state["host"],
state["port"],
update_state=False
)
else: # pragma: nocover else: # pragma: nocover
raise RuntimeError("Unknown State: %s" % state["state"]) raise RuntimeError("Unknown State: %s" % state["state"])
@ -1138,11 +1181,11 @@ class HTTPHandler(ProtocolHandler):
self.c.log(message_debug, level="debug") self.c.log(message_debug, level="debug")
if flow: if flow:
# TODO: no flows without request or with both request and response at the moment. # TODO: no flows without request or with both request and response
# at the moment.
if flow.request and not flow.response: if flow.request and not flow.response:
flow.error = Error(message or message_debug) flow.error = Error(message or message_debug)
self.c.channel.ask("error", flow) self.c.channel.ask("error", flow)
try: try:
code = getattr(error, "code", 502) code = getattr(error, "code", 502)
headers = getattr(error, "headers", None) headers = getattr(error, "headers", None)
@ -1156,12 +1199,22 @@ class HTTPHandler(ProtocolHandler):
def send_error(self, code, message, headers): def send_error(self, code, message, headers):
response = http_status.RESPONSES.get(code, "Unknown") response = http_status.RESPONSES.get(code, "Unknown")
html_content = '<html><head>\n<title>%d %s</title>\n</head>\n<body>\n%s\n</body>\n</html>' % \ html_content = """
(code, response, message) <html>
<head>
<title>%d %s</title>
</head>
<body %s</body>
</html>
""" % (code, response, message)
self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response)) self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response))
self.c.client_conn.wfile.write("Server: %s\r\n" % self.c.config.server_version) self.c.client_conn.wfile.write(
"Server: %s\r\n" % self.c.config.server_version
)
self.c.client_conn.wfile.write("Content-type: text/html\r\n") self.c.client_conn.wfile.write("Content-type: text/html\r\n")
self.c.client_conn.wfile.write("Content-Length: %d\r\n" % len(html_content)) self.c.client_conn.wfile.write(
"Content-Length: %d\r\n" % len(html_content)
)
if headers: if headers:
for key, value in headers.items(): for key, value in headers.items():
self.c.client_conn.wfile.write("%s: %s\r\n" % (key, value)) self.c.client_conn.wfile.write("%s: %s\r\n" % (key, value))
@ -1201,11 +1254,15 @@ class HTTPHandler(ProtocolHandler):
# Now we can process the request. # Now we can process the request.
if request.form_in == "authority": if request.form_in == "authority":
if self.c.client_conn.ssl_established: if self.c.client_conn.ssl_established:
raise http.HttpError(400, "Must not CONNECT on already encrypted connection") raise http.HttpError(
400,
"Must not CONNECT on already encrypted connection"
)
if self.c.config.mode == "regular": if self.c.config.mode == "regular":
self.c.set_server_address((request.host, request.port)) self.c.set_server_address((request.host, request.port))
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow # Update server_conn attribute on the flow
flow.server_conn = self.c.server_conn
self.c.establish_server_connection() self.c.establish_server_connection()
self.c.client_conn.send( self.c.client_conn.send(
'HTTP/1.1 200 Connection established\r\n' + 'HTTP/1.1 200 Connection established\r\n' +
@ -1217,7 +1274,9 @@ class HTTPHandler(ProtocolHandler):
elif self.c.config.mode == "upstream": elif self.c.config.mode == "upstream":
return None return None
else: else:
pass # CONNECT should never occur if we don't expect absolute-form requests # CONNECT should never occur if we don't expect absolute-form
# requests
pass
elif request.form_in == self.expected_form_in: elif request.form_in == self.expected_form_in:
@ -1225,61 +1284,77 @@ class HTTPHandler(ProtocolHandler):
if request.form_in == "absolute": if request.form_in == "absolute":
if request.scheme != "http": if request.scheme != "http":
raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme) raise http.HttpError(
400,
"Invalid request scheme: %s" % request.scheme
)
if self.c.config.mode == "regular": if self.c.config.mode == "regular":
# Update info so that an inline script sees the correct value at flow.server_conn # Update info so that an inline script sees the correct
# value at flow.server_conn
self.c.set_server_address((request.host, request.port)) self.c.set_server_address((request.host, request.port))
flow.server_conn = self.c.server_conn flow.server_conn = self.c.server_conn
return None return None
raise http.HttpError(
raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" % 400, "Invalid HTTP request form (expected: %s, got: %s)" % (
(self.expected_form_in, request.form_in)) self.expected_form_in, request.form_in
)
)
def process_server_address(self, flow): def process_server_address(self, flow):
# Depending on the proxy mode, server handling is entirely different # Depending on the proxy mode, server handling is entirely different
# We provide a mostly unified API to the user, which needs to be unfiddled here # We provide a mostly unified API to the user, which needs to be
# unfiddled here
# ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 )
address = netlib.tcp.Address((flow.request.host, flow.request.port)) address = netlib.tcp.Address((flow.request.host, flow.request.port))
ssl = (flow.request.scheme == "https") ssl = (flow.request.scheme == "https")
if self.c.config.mode == "upstream": if self.c.config.mode == "upstream":
# The connection to the upstream proxy may have a state we may need
# The connection to the upstream proxy may have a state we may need to take into account. # to take into account.
connected_to = None connected_to = None
for s in flow.server_conn.state: for s in flow.server_conn.state:
if s[0] == "http" and s[1]["state"] == "connect": if s[0] == "http" and s[1]["state"] == "connect":
connected_to = tcp.Address((s[1]["host"], s[1]["port"])) connected_to = tcp.Address((s[1]["host"], s[1]["port"]))
# We need to reconnect if the current flow either requires a (possibly impossible) # We need to reconnect if the current flow either requires a
# change to the connection state, e.g. the host has changed but we already CONNECTed somewhere else. # (possibly impossible) change to the connection state, e.g. the
# host has changed but we already CONNECTed somewhere else.
needs_server_change = ( needs_server_change = (
ssl != self.c.server_conn.ssl_established ssl != self.c.server_conn.ssl_established
or or
(connected_to and address != connected_to) # HTTP proxying is "stateless", CONNECT isn't. # HTTP proxying is "stateless", CONNECT isn't.
(connected_to and address != connected_to)
) )
if needs_server_change: if needs_server_change:
# force create new connection to the proxy server to reset state # force create new connection to the proxy server to reset state
self.live.change_server(self.c.server_conn.address, force=True) self.live.change_server(self.c.server_conn.address, force=True)
if ssl: if ssl:
send_connect_request(self.c.server_conn, address.host, address.port) send_connect_request(
self.c.server_conn,
address.host,
address.port
)
self.c.establish_ssl(server=True) self.c.establish_ssl(server=True)
else: else:
# If we're not in upstream mode, we just want to update the host and possibly establish TLS. # If we're not in upstream mode, we just want to update the host and
self.live.change_server(address, ssl=ssl) # this is a no op if the addresses match. # possibly establish TLS. This is a no op if the addresses match.
self.live.change_server(address, ssl=ssl)
flow.server_conn = self.c.server_conn flow.server_conn = self.c.server_conn
def send_response_to_client(self, flow): def send_response_to_client(self, flow):
if not flow.response.stream: if not flow.response.stream:
# no streaming: # no streaming:
# we already received the full response from the server and can send it to the client straight away. # we already received the full response from the server and can send
# it to the client straight away.
self.c.client_conn.send(flow.response.assemble()) self.c.client_conn.send(flow.response.assemble())
else: else:
# streaming: # streaming:
# First send the headers and then transfer the response incrementally: # First send the headers and then transfer the response
# incrementally:
h = flow.response._assemble_head(preserve_transfer_encoding=True) h = flow.response._assemble_head(preserve_transfer_encoding=True)
self.c.client_conn.send(h) self.c.client_conn.send(h)
for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
@ -1293,7 +1368,8 @@ class HTTPHandler(ProtocolHandler):
def check_close_connection(self, flow): def check_close_connection(self, flow):
""" """
Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so. Checks if the connection should be closed depending on the HTTP
semantics. Returns True, if so.
""" """
close_connection = ( close_connection = (
http.connection_close(flow.request.httpversion, flow.request.headers) or http.connection_close(flow.request.httpversion, flow.request.headers) or

4
test/fuzzing/straight_stream
View File

@ -1,6 +1,6 @@
mitmdump: $MITMDUMP -q --stream 1 mitmdump: $MITMDUMP -q --stream 1
pathod: $PATHOD -q pathod: $PATHOD -q
#pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns
pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 /tmp/err #pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 /tmp/err

2
test/fuzzing/straight_stream_patterns
View File

@ -3,3 +3,5 @@ get:'http://localhost:9999/p/':s'200:b"foo"':ir,'a'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'9' get:'http://localhost:9999/p/':s'200:b"foo"':ir,'9'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,':' get:'http://localhost:9999/p/':s'200:b"foo"':ir,':'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'"' get:'http://localhost:9999/p/':s'200:b"foo"':ir,'"'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'-'
get:'http://localhost:9999/p/':s'200:b"foo"':dr