mitmproxy/libmproxy/protocol2/http.py

451 lines
17 KiB
Python
Raw Normal View History

2015-08-11 18:27:34 +00:00
from __future__ import (absolute_import, print_function, division)
2015-08-14 08:41:11 +00:00
from .. import version
from ..exceptions import InvalidCredentials, HttpException, ProtocolException
2015-08-16 21:25:02 +00:00
from .layer import Layer
2015-08-14 14:49:52 +00:00
from libmproxy import utils
2015-08-15 18:20:46 +00:00
from .messages import SetServer, Connect, Reconnect, Kill
2015-08-14 08:41:11 +00:00
from libmproxy.protocol import KILL
2015-08-11 18:27:34 +00:00
from libmproxy.protocol.http import HTTPFlow
2015-08-14 08:41:11 +00:00
from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest
2015-08-11 18:27:34 +00:00
from netlib import tcp
2015-08-18 12:15:08 +00:00
from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError
2015-08-14 14:49:52 +00:00
from netlib.http.semantics import CONTENT_MISSING
2015-08-11 18:27:34 +00:00
from netlib import odict
2015-08-16 21:25:02 +00:00
from netlib.tcp import NetLibError, Address
from netlib.http.http1 import HTTP1Protocol
from netlib.http.http2 import HTTP2Protocol
2015-08-18 12:15:08 +00:00
# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite.
class Http1Layer(Layer):
def __init__(self, ctx, mode):
super(Http1Layer, self).__init__(ctx)
self.mode = mode
self.client_protocol = HTTP1Protocol(self.client_conn)
self.server_protocol = HTTP1Protocol(self.server_conn)
def __call__(self):
layer = HttpLayer(self, self.mode)
for message in layer():
yield message
2015-08-18 12:15:08 +00:00
self.server_protocol = HTTP1Protocol(self.server_conn)
2015-08-11 18:27:34 +00:00
class Http2Layer(Layer):
def __init__(self, ctx, mode):
super(Http2Layer, self).__init__(ctx)
self.mode = mode
self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True)
self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False)
def __call__(self):
layer = HttpLayer(self, self.mode)
for message in layer():
yield message
2015-08-18 12:15:08 +00:00
self.server_protocol = HTTP1Protocol(self.server_conn)
2015-08-11 18:27:34 +00:00
2015-08-14 08:41:11 +00:00
def make_error_response(status_code, message, headers=None):
2015-08-11 18:27:34 +00:00
response = status_codes.RESPONSES.get(status_code, "Unknown")
body = """
<html>
<head>
<title>%d %s</title>
</head>
<body>%s</body>
</html>
""".strip() % (status_code, response, message)
2015-08-14 08:41:11 +00:00
if not headers:
headers = odict.ODictCaseless()
2015-08-11 18:27:34 +00:00
headers["Server"] = [version.NAMEVERSION]
headers["Connection"] = ["close"]
headers["Content-Length"] = [len(body)]
headers["Content-Type"] = ["text/html"]
2015-08-14 08:41:11 +00:00
return HTTPResponse(
(1, 1), # FIXME: Should be a string.
2015-08-11 18:27:34 +00:00
status_code,
response,
headers,
body,
)
2015-08-14 14:49:52 +00:00
2015-08-14 08:41:11 +00:00
def make_connect_request(address):
2015-08-16 21:25:02 +00:00
address = Address.wrap(address)
2015-08-14 08:41:11 +00:00
return HTTPRequest(
2015-08-14 14:49:52 +00:00
"authority", "CONNECT", None, address.host, address.port, None, (1, 1),
2015-08-14 08:41:11 +00:00
odict.ODictCaseless(), ""
)
2015-08-14 14:49:52 +00:00
2015-08-14 08:41:11 +00:00
def make_connect_response(httpversion):
headers = odict.ODictCaseless([
["Content-Length", "0"],
["Proxy-Agent", version.NAMEVERSION]
])
return HTTPResponse(
httpversion,
200,
"Connection established",
headers,
"",
)
2015-08-11 18:27:34 +00:00
2015-08-16 21:25:02 +00:00
class ConnectServerConnection(object):
"""
"Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy.
"""
2015-08-18 12:15:08 +00:00
2015-08-16 21:25:02 +00:00
def __init__(self, address, ctx):
self.address = tcp.Address.wrap(address)
self._ctx = ctx
@property
def via(self):
return self._ctx.server_conn
def __getattr__(self, item):
return getattr(self.via, item)
2015-08-15 14:26:12 +00:00
class HttpLayer(Layer):
2015-08-11 18:27:34 +00:00
"""
HTTP 1 Layer
"""
2015-08-15 18:20:46 +00:00
def __init__(self, ctx, mode):
2015-08-11 18:27:34 +00:00
super(HttpLayer, self).__init__(ctx)
2015-08-15 18:20:46 +00:00
self.mode = mode
2015-08-11 18:27:34 +00:00
def __call__(self):
while True:
try:
2015-08-18 12:15:08 +00:00
flow = HTTPFlow(self.client_conn, self.server_conn, live=True)
2015-08-16 10:43:15 +00:00
try:
request = HTTPRequest.from_protocol(
self.client_protocol,
2015-08-16 10:43:15 +00:00
body_size_limit=self.config.body_size_limit
)
except tcp.NetLibError:
# don't throw an error for disconnects that happen
# before/between requests.
return
self.log("request", "debug", [repr(request)])
# Handle Proxy Authentication
self.authenticate(request)
# Regular Proxy Mode: Handle CONNECT
if self.mode == "regular" and request.form_in == "authority":
2015-08-16 21:25:02 +00:00
for message in self.handle_regular_mode_connect(request):
yield message
2015-08-16 10:43:15 +00:00
return
# Make sure that the incoming request matches our expectations
self.validate_request(request)
flow.request = request
for message in self.process_request_hook(flow):
2015-08-14 14:49:52 +00:00
yield message
2015-08-16 10:43:15 +00:00
if not flow.response:
for message in self.establish_server_connection(flow):
yield message
for message in self.get_response_from_server(flow):
yield message
self.send_response_to_client(flow)
2015-08-14 14:49:52 +00:00
2015-08-16 10:43:15 +00:00
if self.check_close_connection(flow):
return
2015-08-14 14:49:52 +00:00
2015-08-18 12:15:08 +00:00
# TODO: Implement HTTP Upgrade
2015-08-16 21:25:02 +00:00
# Upstream Proxy Mode: Handle CONNECT
2015-08-16 10:43:15 +00:00
if flow.request.form_in == "authority" and flow.response.code == 200:
2015-08-16 21:25:02 +00:00
for message in self.handle_upstream_mode_connect(flow.request.copy()):
yield message
return
2015-08-18 12:15:08 +00:00
except (HttpErrorConnClosed, NetLibError, HttpError) as e:
self.send_to_client(make_error_response(
getattr(e, "code", 502),
repr(e)
))
2015-08-16 10:43:15 +00:00
raise ProtocolException(repr(e), e)
2015-08-18 12:15:08 +00:00
finally:
flow.live = False
2015-08-14 14:49:52 +00:00
2015-08-16 21:25:02 +00:00
def handle_regular_mode_connect(self, request):
yield SetServer((request.host, request.port), False, None)
self.send_to_client(make_connect_response(request.httpversion))
layer = self.ctx.next_layer(self)
for message in layer():
yield message
def handle_upstream_mode_connect(self, connect_request):
layer = self.ctx.next_layer(self)
self.server_conn = ConnectServerConnection((connect_request.host, connect_request.port), self.ctx)
for message in layer():
if message == Connect:
if not self.server_conn:
yield message
self.send_to_server(connect_request)
else:
pass # swallow the message
elif message == Reconnect:
yield message
self.send_to_server(connect_request)
elif message == SetServer:
if message.depth == 1:
if self.ctx.server_conn:
yield Reconnect()
connect_request.host = message.address.host
connect_request.port = message.address.port
self.server_conn.address = message.address
else:
message.depth -= 1
yield message
else:
yield message
2015-08-14 14:49:52 +00:00
def check_close_connection(self, flow):
"""
Checks if the connection should be closed depending on the HTTP
semantics. Returns True, if so.
"""
# TODO: add logic for HTTP/2
close_connection = (
http1.HTTP1Protocol.connection_close(
flow.request.httpversion,
flow.request.headers
) or http1.HTTP1Protocol.connection_close(
flow.response.httpversion,
flow.response.headers
) or http1.HTTP1Protocol.expected_http_body_size(
flow.response.headers,
False,
flow.request.method,
flow.response.code) == -1
2015-08-16 10:43:15 +00:00
)
2015-08-14 14:49:52 +00:00
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for
# https://github.com/mitmproxy/mitmproxy/issues/313: Some
# proxies (e.g. Charles) send a CONNECT response with HTTP/1.0
# and no Content-Length header
return False
return close_connection
def send_response_to_client(self, flow):
if not flow.response.stream:
# no streaming:
# we already received the full response from the server and can
# send it to the client straight away.
self.send_to_client(flow.response)
else:
# streaming:
# First send the headers and then transfer the response
# incrementally:
h = self.client_protocol._assemble_response_first_line(flow.response)
2015-08-14 14:49:52 +00:00
self.send_to_client(h + "\r\n")
h = self.client_protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True)
2015-08-14 14:49:52 +00:00
self.send_to_client(h + "\r\n")
chunks = self.client_protocol.read_http_body_chunked(
2015-08-14 14:49:52 +00:00
flow.response.headers,
self.config.body_size_limit,
flow.request.method,
flow.response.code,
False,
4096
)
if callable(flow.response.stream):
chunks = flow.response.stream(chunks)
for chunk in chunks:
for part in chunk:
2015-08-18 12:15:08 +00:00
# TODO: That's going to fail.
2015-08-14 14:49:52 +00:00
self.send_to_client(part)
self.client_conn.wfile.flush()
flow.response.timestamp_end = utils.timestamp()
def get_response_from_server(self, flow):
2015-08-18 12:15:08 +00:00
def get_response():
self.send_to_server(flow.request)
# Only get the headers at first...
flow.response = HTTPResponse.from_protocol(
self.server_protocol,
flow.request.method,
body_size_limit=self.config.body_size_limit,
include_body=False,
)
try:
get_response()
except (tcp.NetLibError, HttpErrorConnClosed) as v:
self.log(
"server communication error: %s" % repr(v),
level="debug"
)
# In any case, we try to reconnect at least once. This is
# necessary because it might be possible that we already
# initiated an upstream connection after clientconnect that
# has already been expired, e.g consider the following event
# log:
# > clientconnect (transparent mode destination known)
# > serverconnect (required for client tls handshake)
# > read n% of large request
# > server detects timeout, disconnects
# > read (100-n)% of large request
# > send large request upstream
yield Reconnect()
get_response()
2015-08-14 14:49:52 +00:00
# call the appropriate script hook - this is an opportunity for an
# inline script to set flow.stream = True
flow = self.channel.ask("responseheaders", flow)
if flow is None or flow == KILL:
yield Kill()
if flow.response.stream and isinstance(self.server_protocol, http1.HTTP1Protocol):
2015-08-14 14:49:52 +00:00
flow.response.content = CONTENT_MISSING
else:
flow.response.content = self.server_protocol.read_http_body(
2015-08-14 14:49:52 +00:00
flow.response.headers,
self.config.body_size_limit,
flow.request.method,
flow.response.code,
False
)
flow.response.timestamp_end = utils.timestamp()
# no further manipulation of self.server_conn beyond this point
# we can safely set it as the final attribute value here.
flow.server_conn = self.server_conn
self.log(
"response",
"debug",
[repr(flow.response)]
)
response_reply = self.channel.ask("response", flow)
if response_reply is None or response_reply == KILL:
yield Kill()
2015-08-14 08:41:11 +00:00
def process_request_hook(self, flow):
# Determine .scheme, .host and .port attributes for inline scripts.
# For absolute-form requests, they are directly given in the request.
# For authority-form requests, we only need to determine the request scheme.
# For relative-form requests, we need to determine host and port as
# well.
if self.mode == "regular":
pass # only absolute-form at this point, nothing to do here.
elif self.mode == "upstream":
if flow.request.form_in == "authority":
flow.request.scheme = "http" # pseudo value
else:
2015-08-16 21:25:02 +00:00
flow.request.host = self.ctx.server_conn.address.host
flow.request.port = self.ctx.server_conn.address.port
2015-08-15 18:20:46 +00:00
flow.request.scheme = "https" if self.server_conn.tls_established else "http"
2015-08-14 08:41:11 +00:00
2015-08-16 21:25:02 +00:00
# TODO: Expose SetServer functionality to inline scripts somehow? (yield_from_callback?)
2015-08-14 14:49:52 +00:00
request_reply = self.channel.ask("request", flow)
2015-08-14 08:41:11 +00:00
if request_reply is None or request_reply == KILL:
2015-08-14 14:49:52 +00:00
yield Kill()
2015-08-14 08:41:11 +00:00
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
return
def establish_server_connection(self, flow):
address = tcp.Address((flow.request.host, flow.request.port))
tls = (flow.request.scheme == "https")
if self.mode == "regular" or self.mode == "transparent":
# If there's an existing connection that doesn't match our expectations, kill it.
2015-08-16 21:25:02 +00:00
if address != self.server_conn.address or tls != self.server_conn.ssl_established:
2015-08-15 18:20:46 +00:00
yield SetServer(address, tls, address.host)
2015-08-14 08:41:11 +00:00
# Establish connection is neccessary.
if not self.server_conn:
yield Connect()
2015-08-16 21:25:02 +00:00
# SetServer is not guaranteed to work with TLS:
2015-08-14 08:41:11 +00:00
# If there's not TlsLayer below which could catch the exception,
# TLS will not be established.
if tls and not self.server_conn.tls_established:
raise ProtocolException("Cannot upgrade to SSL, no TLS layer on the protocol stack.")
else:
2015-08-16 21:25:02 +00:00
if not self.server_conn:
yield Connect()
2015-08-14 08:41:11 +00:00
if tls:
raise HttpException("Cannot change scheme in upstream proxy mode.")
"""
# This is a very ugly (untested) workaround to solve a very ugly problem.
2015-08-16 21:25:02 +00:00
if self.server_conn and self.server_conn.tls_established and not ssl:
2015-08-14 08:41:11 +00:00
yield Reconnect()
elif ssl and not hasattr(self, "connected_to") or self.connected_to != address:
if self.server_conn.tls_established:
yield Reconnect()
self.send_to_server(make_connect_request(address))
tls_layer = TlsLayer(self, False, True)
tls_layer._establish_tls_with_server()
"""
def validate_request(self, request):
if request.form_in == "absolute" and request.scheme != "http":
self.send_resplonse(make_error_response(400, "Invalid request scheme: %s" % request.scheme))
raise HttpException("Invalid request scheme: %s" % request.scheme)
expected_request_forms = {
2015-08-14 14:49:52 +00:00
"regular": ("absolute",), # an authority request would already be handled.
2015-08-14 08:41:11 +00:00
"upstream": ("authority", "absolute"),
2015-08-15 14:26:12 +00:00
"transparent": ("relative",)
2015-08-14 08:41:11 +00:00
}
allowed_request_forms = expected_request_forms[self.mode]
if request.form_in not in allowed_request_forms:
err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
" or ".join(allowed_request_forms), request.form_in
)
self.send_to_client(make_error_response(400, err_message))
raise HttpException(err_message)
2015-08-15 14:26:12 +00:00
if self.mode == "regular":
request.form_out = "relative"
2015-08-14 08:41:11 +00:00
def authenticate(self, request):
2015-08-11 18:27:34 +00:00
if self.config.authenticator:
if self.config.authenticator.authenticate(request.headers):
self.config.authenticator.clean(request.headers)
else:
2015-08-14 08:41:11 +00:00
self.send_to_client(make_error_response(
407,
"Proxy Authentication Required",
2015-08-18 12:15:08 +00:00
odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()])
2015-08-14 08:41:11 +00:00
))
2015-08-11 18:27:34 +00:00
raise InvalidCredentials("Proxy Authentication Required")
2015-08-14 08:41:11 +00:00
def send_to_server(self, message):
self.server_conn.send(self.server_protocol.assemble(message))
2015-08-15 14:26:12 +00:00
2015-08-14 08:41:11 +00:00
def send_to_client(self, message):
# FIXME
# - possibly do some http2 stuff here
self.client_conn.send(self.client_protocol.assemble(message))