[sans-io] docs++

mitmproxy/proxy2/__init__.py

@@ -1,15 +1,20 @@
 """
-Experimental sans-io implementation of mitmproxy's protocol stack.
+This module contains mitmproxy's core network proxy.
 
-Most important primitives:
+The most important primitives are:
-    - layers: represent protocol layers, e.g. one for tcp, tls, and so on. Layers are nested, so
+
+    - Layers: represent protocol layers, e.g. one for TCP, TLS, and so on. Layers are nested, so
       a typical configuration might be ReverseProxy/TLS/TCP.
-    - server: the proxy server does all IO and communication with the mitmproxy master.
+      Most importantly, layers are implemented using the sans-io pattern (https://sans-io.readthedocs.io/).
-      It creates the top/outermost layer for each incoming client connection.
+      This means that calls return immediately, their is no blocking sync or async code.
-    - events: When IO actions occur at the proxy server, they are passed down to the top layer as events.
+    - Server: the proxy server handles all I/O. This is implemented using asyncio, but could be done any other way.
-    - commands: In the other direction, layers can emit commands to higher layers or the proxy server.
+      The ConnectionHandler is subclassed in the Proxyserver addon, which handles the communication with the
-      This is used to e.g. send data, request for new connections to be opened, or to use mitmproxy's
+      rest of mitmproxy.
-      script hooks.
+    - Events: When I/O actions occur at the proxy server, they are passed to the outermost layer as events,
-    - context: The context is the connection context each layer is provided with. This is still very
+      e.g. "DataReceived" or "ConnectionClosed".
-      much WIP, but this should expose stuff like Server Name Indication to lower layers.
+    - Commands: In the other direction, layers can emit commands to higher layers or the proxy server.
+      This is used to e.g. send data, request for new connections to be opened, or to call mitmproxy's
+      event hooks.
+    - Context: The context is the connection context each layer is provided with, which is always a client connection
+      and sometimes also a server connection.
 """

mitmproxy/proxy2/layer.py

@@ -27,6 +27,22 @@ class Paused(NamedTuple):
 
 
 class Layer:
+    """
+    The base class for all protocol layers.
+
+    Layers interface with their child layer(s) by calling .handle_event(event),
+    which returns a list (more precisely: a generator) of commands.
+    Most layers only implement ._handle_event, which is called by the default implementation of .handle_event.
+    The default implementation allows layers to emulate blocking code:
+    When ._handle_event yields a command that has its blocking attribute set to True, .handle_event pauses
+    the execution of ._handle_event and waits until it is called with the corresponding CommandReply. All events
+    encountered in the meantime are buffered and replayed after execution is resumed.
+
+    The result is code that looks like blocking code, but is not blocking:
+
+        def _handle_event(self, event):
+            err = yield OpenConnection(server)  # execution continues here after a connection has been established.
+    """
     __last_debug_message: ClassVar[str] = ""
     context: Context
     _paused: Optional[Paused]

test/mitmproxy/proxy2/test_tunnel.py

@@ -130,33 +130,38 @@ def test_tunnel_handshake_command(tctx: Context, success):
 
 
 def test_tunnel_default_impls(tctx: Context):
-    tctx.server.state = ConnectionState.OPEN
+    """
-    tl = tunnel.TunnelLayer(tctx, tctx.server, Server(None))
+    Some tunnels don't need certain features, so the default behaviour
+    should be to be transparent.
+    """
+    server = Server(None)
+    server.state = ConnectionState.OPEN
+    tl = tunnel.TunnelLayer(tctx, server, tctx.server)
     tl.child_layer = TChildLayer(tctx)
     playbook = Playbook(tl, logs=True)
     assert (
             playbook
             << Log("Got start. Server state: OPEN")
-            >> DataReceived(tctx.server, b"server-hello")
+            >> DataReceived(server, b"server-hello")
-            << SendData(tctx.server, b"server-hello-reply")
+            << SendData(server, b"server-hello-reply")
     )
     assert tl.tunnel_state is tunnel.TunnelState.OPEN
     assert (
             playbook
-            >> ConnectionClosed(tctx.server)
+            >> ConnectionClosed(server)
             << Log("Got server close.")
-            << CloseConnection(tctx.server)
+            << CloseConnection(server)
     )
     assert tl.tunnel_state is tunnel.TunnelState.CLOSED
 
     assert (
             playbook
             >> DataReceived(tctx.client, b"open")
-            << OpenConnection(tctx.server)
+            << OpenConnection(server)
             >> reply(None)
             << Log("Opened: err=None. Server state: OPEN")
-            >> DataReceived(tctx.server, b"half-close")
+            >> DataReceived(server, b"half-close")
-            << CloseConnection(tctx.server, half_close=True)
+            << CloseConnection(server, half_close=True)
     )