Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-27 02:24:18 +00:00
Code Issues Projects Releases Wiki Activity

more work on http layer

Browse Source
This commit is contained in:
Maximilian Hils 2015-08-14 10:41:11 +02:00
parent aef3b626a7
commit 808218f4bc
8 changed files with 202 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libmproxy/protocol2/__init__.py
View File

@ -3,8 +3,8 @@ from .root_context import RootContext
from .socks_proxy import Socks5Proxy
from .reverse_proxy import ReverseProxy
from .http_proxy import HttpProxy, HttpUpstreamProxy
from .rawtcp import TcpLayer
from .rawtcp import RawTcpLayer
__all__ = [
"Socks5Proxy", "TcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy"
"Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy"
]

178
libmproxy/protocol2/http.py
View File

@ -1,17 +1,22 @@
from __future__ import (absolute_import, print_function, division)
from .. import version
from ..exceptions import InvalidCredentials, HttpException, ProtocolException
from .layer import Layer, ServerConnectionMixin
from libmproxy import version
from libmproxy.exceptions import InvalidCredentials
from .messages import ChangeServer, Connect, Reconnect
from .http_proxy import HttpProxy, HttpUpstreamProxy
from libmproxy.protocol import KILL
from libmproxy.protocol.http import HTTPFlow
from libmproxy.protocol.http_wrappers import HTTPResponse
from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest
from libmproxy.protocol2.http_protocol_mock import HTTP1
from libmproxy.protocol2.tls import TlsLayer
from netlib import tcp
from netlib.http import status_codes
from netlib import odict
def send_http_error_response(status_code, message, headers=odict.ODictCaseless()):
def make_error_response(status_code, message, headers=None):
response = status_codes.RESPONSES.get(status_code, "Unknown")
body = """
<html>
@ -22,21 +27,40 @@ def send_http_error_response(status_code, message, headers=odict.ODictCaseless()
</html>
""".strip() % (status_code, response, message)
if not headers:
headers = odict.ODictCaseless()
headers["Server"] = [version.NAMEVERSION]
headers["Connection"] = ["close"]
headers["Content-Length"] = [len(body)]
headers["Content-Type"] = ["text/html"]
resp = HTTPResponse(
(1, 1), # if HTTP/2 is used, this value is ignored anyway
return HTTPResponse(
(1, 1), # FIXME: Should be a string.
status_code,
response,
headers,
body,
)
protocol = self.c.client_conn.protocol or http1.HTTP1Protocol(self.c.client_conn)
self.c.client_conn.send(protocol.assemble(resp))
def make_connect_request(address):
return HTTPRequest(
"authority", "CONNECT", None, address.host, address.port, None, (1,1),
odict.ODictCaseless(), ""
)
def make_connect_response(httpversion):
headers = odict.ODictCaseless([
["Content-Length", "0"],
["Proxy-Agent", version.NAMEVERSION]
])
return HTTPResponse(
httpversion,
200,
"Connection established",
headers,
"",
)
class HttpLayer(Layer, ServerConnectionMixin):
"""
@ -45,11 +69,16 @@ class HttpLayer(Layer, ServerConnectionMixin):
def __init__(self, ctx):
super(HttpLayer, self).__init__(ctx)
self.skip_authentication = False
if any(isinstance(l, HttpProxy) for l in self.layers):
self.mode = "regular"
elif any(isinstance(l, HttpUpstreamProxy) for l in self.layers):
self.mode = "upstream"
else:
# also includes socks or reverse mode, which are handled similarly on this layer.
self.mode = "transparent"
def __call__(self):
while True:
flow = HTTPFlow(self.client_conn, self.server_conn)
try:
request = HTTP1.read_request(
self.client_conn,
@ -62,29 +91,126 @@ class HttpLayer(Layer, ServerConnectionMixin):
self.c.log("request", "debug", [repr(request)])
self.check_authentication(request)
# Handle Proxy Authentication
self.authenticate(request)
# Regular Proxy Mode: Handle CONNECT
if self.mode == "regular" and request.form_in == "authority":
raise NotImplementedError
ret = self.process_request(flow, request)
if ret is True:
continue
if ret is False:
self.server_address = (request.host, request.port)
self.send_to_client(make_connect_response(request.httpversion))
layer = self.ctx.next_layer(self)
for message in layer():
if not self._handle_server_message(message):
yield message
return
def check_authentication(self, request):
# Make sure that the incoming request matches our expectations
self.validate_request(request)
flow = HTTPFlow(self.client_conn, self.server_conn)
flow.request = request
if not self.process_request_hook(flow):
self.log("Connection killed", "info")
return
if not flow.response:
self.establish_server_connection(flow)
def process_request_hook(self, flow):
# Determine .scheme, .host and .port attributes for inline scripts.
# For absolute-form requests, they are directly given in the request.
# For authority-form requests, we only need to determine the request scheme.
# For relative-form requests, we need to determine host and port as
# well.
if self.mode == "regular":
pass # only absolute-form at this point, nothing to do here.
elif self.mode == "upstream":
if flow.request.form_in == "authority":
flow.request.scheme = "http" # pseudo value
else:
flow.request.host = self.ctx.server_address.host
flow.request.port = self.ctx.server_address.port
flow.request.scheme = self.server_conn.tls_established
# TODO: Expose ChangeServer functionality to inline scripts somehow? (yield_from_callback?)
request_reply = self.c.channel.ask("request", flow)
if request_reply is None or request_reply == KILL:
return False
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
return
def establish_server_connection(self, flow):
address = tcp.Address((flow.request.host, flow.request.port))
tls = (flow.request.scheme == "https")
if self.mode == "regular" or self.mode == "transparent":
# If there's an existing connection that doesn't match our expectations, kill it.
if self.server_address != address or tls != self.server_address.ssl_established:
yield ChangeServer(address, tls, address.host)
# Establish connection is neccessary.
if not self.server_conn:
yield Connect()
# ChangeServer is not guaranteed to work with TLS:
# If there's not TlsLayer below which could catch the exception,
# TLS will not be established.
if tls and not self.server_conn.tls_established:
raise ProtocolException("Cannot upgrade to SSL, no TLS layer on the protocol stack.")
else:
if tls:
raise HttpException("Cannot change scheme in upstream proxy mode.")
"""
# This is a very ugly (untested) workaround to solve a very ugly problem.
# FIXME: Check if connected first.
if self.server_conn.tls_established and not ssl:
yield Reconnect()
elif ssl and not hasattr(self, "connected_to") or self.connected_to != address:
if self.server_conn.tls_established:
yield Reconnect()
self.send_to_server(make_connect_request(address))
tls_layer = TlsLayer(self, False, True)
tls_layer._establish_tls_with_server()
"""
def validate_request(self, request):
if request.form_in == "absolute" and request.scheme != "http":
self.send_resplonse(make_error_response(400, "Invalid request scheme: %s" % request.scheme))
raise HttpException("Invalid request scheme: %s" % request.scheme)
expected_request_forms = {
"regular": ("absolute",), # an authority request would already be handled.
"upstream": ("authority", "absolute"),
"transparent": ("regular",)
}
allowed_request_forms = expected_request_forms[self.mode]
if request.form_in not in allowed_request_forms:
err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
" or ".join(allowed_request_forms), request.form_in
)
self.send_to_client(make_error_response(400, err_message))
raise HttpException(err_message)
def authenticate(self, request):
if self.config.authenticator:
if self.config.authenticator.authenticate(request.headers):
self.config.authenticator.clean(request.headers)
else:
self.send_error()
self.send_to_client(make_error_response(
407,
"Proxy Authentication Required",
self.config.authenticator.auth_challenge_headers()
))
raise InvalidCredentials("Proxy Authentication Required")
raise http.HttpAuthenticationError(
self.c.config.authenticator.auth_challenge_headers())
return request.headers
def send_error(self, code, message, headers):
pass
def send_to_server(self, message):
self.server_conn.wfile.wrie(message)
def send_to_client(self, message):
# FIXME
# - possibly do some http2 stuff here
# - fix message assembly.
self.client_conn.wfile.write(message)

5
libmproxy/protocol2/http_proxy.py
View File

@ -4,11 +4,12 @@ from .layer import Layer, ServerConnectionMixin
from .http import HttpLayer
class HttpProxy(Layer):
class HttpProxy(Layer, ServerConnectionMixin):
def __call__(self):
layer = HttpLayer(self)
for message in layer():
yield message
if not self._handle_server_message(message):
yield message
class HttpUpstreamProxy(Layer, ServerConnectionMixin):

11
libmproxy/protocol2/layer.py
View File

@ -16,9 +16,7 @@ Regular proxy, CONNECT request with WebSockets over SSL:
Automated protocol detection by peeking into the buffer:
TransparentModeLayer
AutoLayer
SslLayer
AutoLayer
Http2Layer
Communication between layers is done as follows:
@ -91,6 +89,13 @@ class Layer(_LayerCodeCompletion):
full_msg = "\n".join(full_msg)
self.channel.tell("log", Log(full_msg, level))
@property
def layers(self):
return [self] + self.ctx.layers
def __repr__(self):
return "%s\r\n %s" % (self.__class__.name__, repr(self.ctx))
class ServerConnectionMixin(object):
"""
@ -133,6 +138,8 @@ class ServerConnectionMixin(object):
self.server_conn = None
def _connect(self):
if not self.server_address:
raise ProtocolException("Cannot connect to server, no server address given.")
self.log("serverconnect", "debug", [repr(self.server_address)])
self.server_conn = ServerConnection(self.server_address)
try:

3
libmproxy/protocol2/rawtcp.py
View File

@ -1,4 +1,5 @@
from __future__ import (absolute_import, print_function, division)
import OpenSSL
from ..exceptions import ProtocolException
from ..protocol.tcp import TCPHandler
@ -6,7 +7,7 @@ from .layer import Layer
from .messages import Connect
class TcpLayer(Layer):
class RawTcpLayer(Layer):
def __call__(self):
yield Connect()
tcp_handler = TCPHandler(self)

5
libmproxy/protocol2/reverse_proxy.py
View File

@ -13,7 +13,10 @@ class ReverseProxy(Layer, ServerConnectionMixin):
self._server_tls = server_tls
def __call__(self):
layer = TlsLayer(self, self._client_tls, self._server_tls)
if self._client_tls or self._server_tls:
layer = TlsLayer(self, self._client_tls, self._server_tls)
else:
layer = self.ctx.next_layer(self)
for message in layer():
if not self._handle_server_message(message):
yield message

29
libmproxy/protocol2/root_context.py
View File

@ -1,4 +1,6 @@
from .rawtcp import TcpLayer
from __future__ import (absolute_import, print_function, division)
from .rawtcp import RawTcpLayer
from .tls import TlsLayer
@ -20,13 +22,30 @@ class RootContext(object):
:return: The next layer.
"""
d = top_layer.client_conn.rfile.peek(1)
d = top_layer.client_conn.rfile.peek(3)
# TODO: Handle ignore and tcp passthrough
# TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello
is_tls_client_hello = (
len(d) == 3 and
d[0] == '\x16' and
d[1] == '\x03' and
d[2] in ('\x00', '\x01', '\x02', '\x03')
)
if not d:
return
# TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello
if d[0] == "\x16":
if is_tls_client_hello:
layer = TlsLayer(top_layer, True, True)
else:
layer = TcpLayer(top_layer)
layer = RawTcpLayer(top_layer)
return layer
@property
def layers(self):
return []
def __repr__(self):
return "RootContext"

11
libmproxy/protocol2/tls.py
View File

@ -1,4 +1,5 @@
from __future__ import (absolute_import, print_function, division)
import traceback
from netlib import tcp
@ -99,7 +100,7 @@ class TlsLayer(Layer):
if server_err and not self.client_sni:
raise server_err
def handle_sni(self, connection):
def __handle_sni(self, connection):
"""
This callback gets called during the TLS handshake with the client.
The client has just sent the Sever Name Indication (SNI).
@ -119,7 +120,7 @@ class TlsLayer(Layer):
if self.client_sni:
# Now, change client context to reflect possibly changed certificate:
cert, key, chain_file = self.find_cert()
cert, key, chain_file = self._find_cert()
new_context = self.client_conn.create_ssl_context(
cert, key,
method=self.config.openssl_method_client,
@ -137,13 +138,13 @@ class TlsLayer(Layer):
@yield_from_callback
def _establish_tls_with_client(self):
self.log("Establish TLS with client", "debug")
cert, key, chain_file = self.find_cert()
cert, key, chain_file = self._find_cert()
try:
self.client_conn.convert_to_ssl(
cert, key,
method=self.config.openssl_method_client,
options=self.config.openssl_options_client,
handle_sni=self.handle_sni,
handle_sni=self.__handle_sni,
cipher_list=self.config.ciphers_client,
dhparams=self.config.certstore.dhparams,
chain_file=chain_file
@ -182,7 +183,7 @@ class TlsLayer(Layer):
except tcp.NetLibError as e:
raise ProtocolException(repr(e), e)
def find_cert(self):
def _find_cert(self):
host = self.server_conn.address.host
sans = set()
# Incorporate upstream certificate