Merge pull request #2789 from mhils/move-clienthello

Move ClientHello and related functions to mitmproxy.net.tls
This commit is contained in:
Maximilian Hils 2018-01-13 00:35:36 +01:00 committed by GitHub
commit 37527a1da3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 240 additions and 155 deletions

View File

@ -2,15 +2,20 @@
# then add options to disable certain methods
# https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3
import binascii
import io
import os
import struct
import threading
import typing
from ssl import match_hostname, CertificateError
import certifi
from OpenSSL import SSL
from kaitaistruct import KaitaiStream
from mitmproxy import exceptions, certs
from mitmproxy.contrib.kaitaistruct import tls_client_hello
from mitmproxy.net import check
BASIC_OPTIONS = (
SSL.OP_CIPHER_SERVER_PREFERENCE
@ -189,7 +194,7 @@ def _create_ssl_context(
def create_client_context(
cert: str = None,
sni: str = None,
address: str=None,
address: str = None,
verify: int = SSL.VERIFY_NONE,
**sslctx_kwargs
) -> SSL.Context:
@ -338,3 +343,119 @@ def create_server_context(
SSL._lib.SSL_CTX_set_tmp_dh(context._context, dhparams)
return context
def is_tls_record_magic(d):
"""
Returns:
True, if the passed bytes start with the TLS record magic bytes.
False, otherwise.
"""
d = d[:3]
# TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2
# http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello
return (
len(d) == 3 and
d[0] == 0x16 and
d[1] == 0x03 and
0x0 <= d[2] <= 0x03
)
def get_client_hello(rfile):
"""
Peek into the socket and read all records that contain the initial client hello message.
client_conn:
The :py:class:`client connection <mitmproxy.connections.ClientConnection>`.
Returns:
The raw handshake packet bytes, without TLS record header(s).
"""
client_hello = b""
client_hello_size = 1
offset = 0
while len(client_hello) < client_hello_size:
record_header = rfile.peek(offset + 5)[offset:]
if not is_tls_record_magic(record_header) or len(record_header) < 5:
raise exceptions.TlsProtocolException(
'Expected TLS record, got "%s" instead.' % record_header)
record_size = struct.unpack_from("!H", record_header, 3)[0] + 5
record_body = rfile.peek(offset + record_size)[offset + 5:]
if len(record_body) != record_size - 5:
raise exceptions.TlsProtocolException(
"Unexpected EOF in TLS handshake: %s" % record_body)
client_hello += record_body
offset += record_size
client_hello_size = struct.unpack("!I", b'\x00' + client_hello[1:4])[0] + 4
return client_hello
class ClientHello:
def __init__(self, raw_client_hello):
self._client_hello = tls_client_hello.TlsClientHello(
KaitaiStream(io.BytesIO(raw_client_hello))
)
@property
def cipher_suites(self):
return self._client_hello.cipher_suites.cipher_suites
@property
def sni(self):
if self._client_hello.extensions:
for extension in self._client_hello.extensions.extensions:
is_valid_sni_extension = (
extension.type == 0x00 and
len(extension.body.server_names) == 1 and
extension.body.server_names[0].name_type == 0 and
check.is_valid_host(extension.body.server_names[0].host_name)
)
if is_valid_sni_extension:
return extension.body.server_names[0].host_name.decode("idna")
return None
@property
def alpn_protocols(self):
if self._client_hello.extensions:
for extension in self._client_hello.extensions.extensions:
if extension.type == 0x10:
return list(x.name for x in extension.body.alpn_protocols)
return []
@property
def extensions(self) -> typing.List[typing.Tuple[int, bytes]]:
ret = []
if self._client_hello.extensions:
for extension in self._client_hello.extensions.extensions:
body = getattr(extension, "_raw_body", extension.body)
ret.append((extension.type, body))
return ret
@classmethod
def from_file(cls, client_conn) -> "ClientHello":
"""
Peek into the connection, read the initial client hello and parse it to obtain ALPN values.
client_conn:
The :py:class:`client connection <mitmproxy.connections.ClientConnection>`.
Returns:
:py:class:`client hello <mitmproxy.net.tls.ClientHello>`.
"""
try:
raw_client_hello = get_client_hello(client_conn)[4:] # exclude handshake header.
except exceptions.ProtocolException as e:
raise exceptions.TlsProtocolException('Cannot read raw Client Hello: %s' % repr(e))
try:
return cls(raw_client_hello)
except EOFError as e:
raise exceptions.TlsProtocolException(
'Cannot parse Client Hello: %s, Raw Client Hello: %s' %
(repr(e), binascii.hexlify(raw_client_hello))
)
def __repr__(self):
return "ClientHello(sni: %s, alpn_protocols: %s, cipher_suites: %s)" % \
(self.sni, self.alpn_protocols, self.cipher_suites)

View File

@ -36,13 +36,11 @@ from .http1 import Http1Layer
from .http2 import Http2Layer
from .websocket import WebSocketLayer
from .rawtcp import RawTCPLayer
from .tls import TlsClientHello
from .tls import TlsLayer
from .tls import is_tls_record_magic
__all__ = [
"Layer", "ServerConnectionMixin",
"TlsLayer", "is_tls_record_magic", "TlsClientHello",
"TlsLayer",
"UpstreamConnectLayer",
"HttpLayer",
"Http1Layer",

View File

@ -1,14 +1,9 @@
import struct
from typing import Optional # noqa
from typing import Union
import io
from kaitaistruct import KaitaiStream
from mitmproxy import exceptions
from mitmproxy.contrib.kaitaistruct import tls_client_hello
from mitmproxy.net import tls as net_tls
from mitmproxy.proxy.protocol import base
from mitmproxy.net import check
# taken from https://testssl.sh/openssl-rfc.mappping.html
CIPHER_ID_NAME_MAP = {
@ -200,7 +195,6 @@ CIPHER_ID_NAME_MAP = {
0x080080: 'RC4-64-MD5',
}
# We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default.
# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old
DEFAULT_CLIENT_CIPHERS = (
@ -216,114 +210,7 @@ DEFAULT_CLIENT_CIPHERS = (
)
def is_tls_record_magic(d):
"""
Returns:
True, if the passed bytes start with the TLS record magic bytes.
False, otherwise.
"""
d = d[:3]
# TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2
# http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello
return (
len(d) == 3 and
d[0] == 0x16 and
d[1] == 0x03 and
0x0 <= d[2] <= 0x03
)
def get_client_hello(client_conn):
"""
Peek into the socket and read all records that contain the initial client hello message.
client_conn:
The :py:class:`client connection <mitmproxy.connections.ClientConnection>`.
Returns:
The raw handshake packet bytes, without TLS record header(s).
"""
client_hello = b""
client_hello_size = 1
offset = 0
while len(client_hello) < client_hello_size:
record_header = client_conn.rfile.peek(offset + 5)[offset:]
if not is_tls_record_magic(record_header) or len(record_header) != 5:
raise exceptions.TlsProtocolException('Expected TLS record, got "%s" instead.' % record_header)
record_size = struct.unpack("!H", record_header[3:])[0] + 5
record_body = client_conn.rfile.peek(offset + record_size)[offset + 5:]
if len(record_body) != record_size - 5:
raise exceptions.TlsProtocolException("Unexpected EOF in TLS handshake: %s" % record_body)
client_hello += record_body
offset += record_size
client_hello_size = struct.unpack("!I", b'\x00' + client_hello[1:4])[0] + 4
return client_hello
class TlsClientHello:
def __init__(self, raw_client_hello):
self._client_hello = tls_client_hello.TlsClientHello(KaitaiStream(io.BytesIO(raw_client_hello)))
def raw(self):
return self._client_hello
@property
def cipher_suites(self):
return self._client_hello.cipher_suites.cipher_suites
@property
def sni(self):
if self._client_hello.extensions:
for extension in self._client_hello.extensions.extensions:
is_valid_sni_extension = (
extension.type == 0x00 and
len(extension.body.server_names) == 1 and
extension.body.server_names[0].name_type == 0 and
check.is_valid_host(extension.body.server_names[0].host_name)
)
if is_valid_sni_extension:
return extension.body.server_names[0].host_name.decode("idna")
return None
@property
def alpn_protocols(self):
if self._client_hello.extensions:
for extension in self._client_hello.extensions.extensions:
if extension.type == 0x10:
return list(x.name for x in extension.body.alpn_protocols)
return []
@classmethod
def from_client_conn(cls, client_conn):
"""
Peek into the connection, read the initial client hello and parse it to obtain ALPN values.
client_conn:
The :py:class:`client connection <mitmproxy.connections.ClientConnection>`.
Returns:
:py:class:`client hello <mitmproxy.proxy.protocol.tls.TlsClientHello>`.
"""
try:
raw_client_hello = get_client_hello(client_conn)[4:] # exclude handshake header.
except exceptions.ProtocolException as e:
raise exceptions.TlsProtocolException('Cannot read raw Client Hello: %s' % repr(e))
try:
return cls(raw_client_hello)
except EOFError as e:
raise exceptions.TlsProtocolException(
'Cannot parse Client Hello: %s, Raw Client Hello: %s' %
(repr(e), raw_client_hello.encode("hex"))
)
def __repr__(self):
return "TlsClientHello( sni: %s alpn_protocols: %s, cipher_suites: %s)" % \
(self.sni, self.alpn_protocols, self.cipher_suites)
class TlsLayer(base.Layer):
"""
The TLS layer implements transparent TLS connections.
@ -334,13 +221,13 @@ class TlsLayer(base.Layer):
the server connection.
"""
def __init__(self, ctx, client_tls, server_tls, custom_server_sni = None):
def __init__(self, ctx, client_tls, server_tls, custom_server_sni=None):
super().__init__(ctx)
self._client_tls = client_tls
self._server_tls = server_tls
self._custom_server_sni = custom_server_sni
self._client_hello = None # type: Optional[TlsClientHello]
self._client_hello = None # type: Optional[net_tls.ClientHello]
def __call__(self):
"""
@ -355,7 +242,7 @@ class TlsLayer(base.Layer):
if self._client_tls:
# Peek into the connection, read the initial client hello and parse it to obtain SNI and ALPN values.
try:
self._client_hello = TlsClientHello.from_client_conn(self.client_conn)
self._client_hello = net_tls.ClientHello.from_file(self.client_conn.rfile)
except exceptions.TlsProtocolException as e:
self.log("Cannot parse Client Hello: %s" % repr(e), "error")
@ -414,7 +301,7 @@ class TlsLayer(base.Layer):
if self._server_tls and not self.server_conn.tls_established:
self._establish_tls_with_server()
def set_server_tls(self, server_tls: bool, sni: Union[str, None, bool]=None) -> None:
def set_server_tls(self, server_tls: bool, sni: Union[str, None, bool] = None) -> None:
"""
Set the TLS settings for the next server connection that will be established.
This function will not alter an existing connection.
@ -519,8 +406,10 @@ class TlsLayer(base.Layer):
# We only support http/1.1 and h2.
# If the server only supports spdy (next to http/1.1), it may select that
# and mitmproxy would enter TCP passthrough mode, which we want to avoid.
alpn = [x for x in self._client_hello.alpn_protocols if
not (x.startswith(b"h2-") or x.startswith(b"spdy"))]
alpn = [
x for x in self._client_hello.alpn_protocols if
not (x.startswith(b"h2-") or x.startswith(b"spdy"))
]
if alpn and b"h2" in alpn and not self.config.options.http2:
alpn.remove(b"h2")

View File

@ -1,5 +1,6 @@
from mitmproxy import log
from mitmproxy import exceptions
from mitmproxy.net import tls
from mitmproxy.proxy import protocol
from mitmproxy.proxy import modes
from mitmproxy.proxy.protocol import http
@ -45,14 +46,14 @@ class RootContext:
d = top_layer.client_conn.rfile.peek(3)
except exceptions.TcpException as e:
raise exceptions.ProtocolException(str(e))
client_tls = protocol.is_tls_record_magic(d)
client_tls = tls.is_tls_record_magic(d)
# 1. check for --ignore
if self.config.check_ignore:
ignore = self.config.check_ignore(top_layer.server_conn.address)
if not ignore and client_tls:
try:
client_hello = protocol.TlsClientHello.from_client_conn(self.client_conn)
client_hello = tls.ClientHello.from_file(self.client_conn.rfile)
except exceptions.TlsProtocolException as e:
self.log("Cannot parse Client Hello: %s" % repr(e), "error")
else:
@ -76,10 +77,10 @@ class RootContext:
# if the user manually sets a scheme for connect requests, we use this to decide if we
# want TLS or not.
if top_layer.connect_request.scheme:
tls = top_layer.connect_request.scheme == "https"
server_tls = top_layer.connect_request.scheme == "https"
else:
tls = client_tls
return protocol.TlsLayer(top_layer, client_tls, tls)
server_tls = client_tls
return protocol.TlsLayer(top_layer, client_tls, server_tls)
# 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed.
if isinstance(top_layer, protocol.TlsLayer):

View File

@ -1,3 +1,5 @@
import io
import pytest
from mitmproxy import exceptions
@ -6,6 +8,17 @@ from mitmproxy.net.tcp import TCPClient
from test.mitmproxy.net.test_tcp import EchoHandler
from . import tservers
CLIENT_HELLO_NO_EXTENSIONS = bytes.fromhex(
"03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637"
"78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000"
"61006200640100"
)
FULL_CLIENT_HELLO_NO_EXTENSIONS = (
b"\x16\x03\x03\x00\x65" # record layer
b"\x01\x00\x00\x61" + # handshake header
CLIENT_HELLO_NO_EXTENSIONS
)
class TestMasterSecretLogger(tservers.ServerTestBase):
handler = EchoHandler
@ -53,3 +66,92 @@ class TestTLSInvalid:
with pytest.raises(exceptions.TlsException, match="ALPN error"):
tls.create_client_context(alpn_select="foo", alpn_select_callback="bar")
def test_is_record_magic():
assert not tls.is_tls_record_magic(b"POST /")
assert not tls.is_tls_record_magic(b"\x16\x03")
assert not tls.is_tls_record_magic(b"\x16\x03\x04")
assert tls.is_tls_record_magic(b"\x16\x03\x00")
assert tls.is_tls_record_magic(b"\x16\x03\x01")
assert tls.is_tls_record_magic(b"\x16\x03\x02")
assert tls.is_tls_record_magic(b"\x16\x03\x03")
def test_get_client_hello():
rfile = io.BufferedReader(io.BytesIO(
FULL_CLIENT_HELLO_NO_EXTENSIONS
))
assert tls.get_client_hello(rfile)
rfile = io.BufferedReader(io.BytesIO(
FULL_CLIENT_HELLO_NO_EXTENSIONS[:30]
))
with pytest.raises(exceptions.TlsProtocolException, message="Unexpected EOF"):
tls.get_client_hello(rfile)
rfile = io.BufferedReader(io.BytesIO(
b"GET /"
))
with pytest.raises(exceptions.TlsProtocolException, message="Expected TLS record"):
tls.get_client_hello(rfile)
class TestClientHello:
def test_no_extensions(self):
c = tls.ClientHello(CLIENT_HELLO_NO_EXTENSIONS)
assert repr(c)
assert c.sni is None
assert c.cipher_suites == [53, 47, 10, 5, 4, 9, 3, 6, 8, 96, 97, 98, 100]
assert c.alpn_protocols == []
assert c.extensions == []
def test_extensions(self):
data = bytes.fromhex(
"03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030"
"cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65"
"78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501"
"00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00"
"170018"
)
c = tls.ClientHello(data)
assert repr(c)
assert c.sni == 'example.com'
assert c.cipher_suites == [
49195, 49199, 49196, 49200, 52393, 52392, 52244, 52243, 49161,
49171, 49162, 49172, 156, 157, 47, 53, 10
]
assert c.alpn_protocols == [b'h2', b'http/1.1']
assert c.extensions == [
(65281, b'\x00'),
(0, b'\x00\x0e\x00\x00\x0bexample.com'),
(23, b''),
(35, b''),
(13, b'\x00\x10\x06\x01\x06\x03\x05\x01\x05\x03\x04\x01\x04\x03\x02\x01\x02\x03'),
(5, b'\x01\x00\x00\x00\x00'),
(18, b''),
(16, b'\x00\x0c\x02h2\x08http/1.1'),
(30032, b''),
(11, b'\x01\x00'),
(10, b'\x00\x06\x00\x1d\x00\x17\x00\x18')
]
def test_from_conn(self):
rfile = io.BufferedReader(io.BytesIO(
FULL_CLIENT_HELLO_NO_EXTENSIONS
))
assert tls.ClientHello.from_file(rfile)
rfile = io.BufferedReader(io.BytesIO(
b""
))
with pytest.raises(exceptions.TlsProtocolException):
tls.ClientHello.from_file(rfile)
rfile = io.BufferedReader(io.BytesIO(
b"\x16\x03\x03\x00\x07" # record layer
b"\x01\x00\x00\x03" + # handshake header
b"foo"
))
with pytest.raises(exceptions.TlsProtocolException, message='Cannot parse Client Hello'):
tls.ClientHello.from_file(rfile)

View File

@ -1,26 +0,0 @@
from mitmproxy.proxy.protocol.tls import TlsClientHello
class TestClientHello:
def test_no_extensions(self):
data = bytes.fromhex(
"03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637"
"78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000"
"61006200640100"
)
c = TlsClientHello(data)
assert c.sni is None
assert c.alpn_protocols == []
def test_extensions(self):
data = bytes.fromhex(
"03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030"
"cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65"
"78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501"
"00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00"
"170018"
)
c = TlsClientHello(data)
assert c.sni == 'example.com'
assert c.alpn_protocols == [b'h2', b'http/1.1']