Merge pull request #2791 from mhils/tls-extensions

Store ClientHello extensions with client connection

mitmproxy/connections.py

@@ -1,11 +1,13 @@
 import time
 
 import os
+import typing
 import uuid
 
-from mitmproxy import stateobject
+from mitmproxy import stateobject, exceptions
 from mitmproxy import certs
 from mitmproxy.net import tcp
+from mitmproxy.net import tls
 from mitmproxy.utils import strutils
 
 
@@ -26,6 +28,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         cipher_name: The current used cipher
         alpn_proto_negotiated: The negotiated application protocol
         tls_version: TLS version
+        tls_extensions: TLS ClientHello extensions
     """
 
     def __init__(self, client_connection, address, server):
@@ -51,6 +54,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         self.cipher_name = None
         self.alpn_proto_negotiated = None
         self.tls_version = None
+        self.tls_extensions = None
 
     def connected(self):
         return bool(self.connection) and not self.finished
@@ -96,6 +100,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         cipher_name=str,
         alpn_proto_negotiated=bytes,
         tls_version=str,
+        tls_extensions=typing.List[typing.Tuple[int, bytes]],
     )
 
     def send(self, message):
@@ -125,9 +130,19 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
             cipher_name=None,
             alpn_proto_negotiated=None,
             tls_version=None,
+            tls_extensions=None,
         ))
 
     def convert_to_tls(self, cert, *args, **kwargs):
+        # Unfortunately OpenSSL provides no way to expose all TLS extensions, so we do this dance
+        # here and use our Kaitai parser.
+        try:
+            client_hello = tls.ClientHello.from_file(self.rfile)
+        except exceptions.TlsProtocolException:  # pragma: no cover
+            pass  # if this fails, we don't want everything to go down.
+        else:
+            self.tls_extensions = client_hello.extensions
+
         super().convert_to_tls(cert, *args, **kwargs)
         self.timestamp_tls_setup = time.time()
         self.mitmcert = cert

mitmproxy/io/compat.py

@@ -166,11 +166,10 @@ def convert_5_6(data):
     return data
 
 
-# def convert_6_7(data):
-#     data["version"] = 7
-#     # Your changes here!
-#     # Make sure to also increment FLOW_FORMAT_VERSION.
-#     return data
+def convert_6_7(data):
+    data["version"] = 7
+    data["client_conn"]["tls_extensions"] = None
+    return data
 
 
 def _convert_dict_keys(o: Any) -> Any:
@@ -226,7 +225,7 @@ converters = {
     (3, 0): convert_300_4,
     4: convert_4_5,
     5: convert_5_6,
-    # 6: convert_6_7,
+    6: convert_6_7,
 }
 
 

mitmproxy/test/tflow.py

@@ -165,6 +165,7 @@ def tclient_conn():
         cipher_name="cipher",
         alpn_proto_negotiated=b"http/1.1",
         tls_version="TLSv1.2",
+        tls_extensions=[(0x00, bytes.fromhex("000e00000b6578616d"))],
     ))
     c.reply = controller.DummyReply()
     c.rfile = io.BytesIO()

mitmproxy/tools/web/app.py

@@ -43,6 +43,8 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict:
             continue
         f[conn]["alpn_proto_negotiated"] = \
             f[conn]["alpn_proto_negotiated"].decode(errors="backslashreplace")
+    # There are some bytes in here as well, let's skip it until we have them in the UI.
+    f["client_conn"].pop("tls_extensions", None)
     if flow.error:
         f["error"] = flow.error.get_state()
 

mitmproxy/version.py

@@ -9,7 +9,7 @@ MITMPROXY = "mitmproxy " + VERSION
 
 # Serialization format version. This is displayed nowhere, it just needs to be incremented by one
 # for each change in the file format.
-FLOW_FORMAT_VERSION = 6
+FLOW_FORMAT_VERSION = 7
 
 
 def get_version(dev: bool = False, build: bool = False, refresh: bool = False) -> str:

test/mitmproxy/net/test_tls.py

@@ -136,7 +136,7 @@ class TestClientHello:
             (10, b'\x00\x06\x00\x1d\x00\x17\x00\x18')
         ]
 
-    def test_from_conn(self):
+    def test_from_file(self):
         rfile = io.BufferedReader(io.BytesIO(
             FULL_CLIENT_HELLO_NO_EXTENSIONS
         ))

test/mitmproxy/proxy/test_server.py

@@ -723,12 +723,13 @@ class TestProxy(tservers.HTTPProxyTest):
 class TestProxySSL(tservers.HTTPProxyTest):
     ssl = True
 
-    def test_request_ssl_setup_timestamp_presence(self):
+    def test_request_tls_attribute_presence(self):
         # tests that the ssl timestamp is present when ssl is used
         f = self.pathod("304:b@10k")
         assert f.status_code == 304
         first_flow = self.master.state.flows[0]
         assert first_flow.server_conn.timestamp_tls_setup
+        assert first_flow.client_conn.tls_extensions
 
     def test_via(self):
         # tests that the ssl timestamp is present when ssl is used