docs: better data structure docs

docs/Makefile

@@ -192,4 +192,4 @@ pseudoxml:
 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
 
 livehtml:
-	sphinx-autobuild -b html -z '../mitmproxy' -z '../../netlib/netlib' -r '___jb_(old|bak)___$$' $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	sphinx-autobuild -b html -z '../mitmproxy' -r '___jb_(old|bak|tmp)___$$' $(ALLSPHINXOPTS) $(BUILDDIR)/html

docs/dev/models.rst

@@ -1,7 +1,11 @@
 .. _models:
 
-Models
-======
+Datastructures
+==============
+
+.. automodule:: mitmproxy.models
+    :members: HTTPFlow, HTTPRequest, HTTPResponse
+
 
 .. automodule:: netlib.http
 
@@ -56,4 +60,4 @@ Models
 
 .. automodule:: mitmproxy.models
     :show-inheritance:
-    :members: HTTPFlow, Error, ClientConnection, ServerConnection
+    :members: Error, ServerConnection, ClientConnection

docs/dev/testing.rst

@@ -10,7 +10,7 @@ suitable extension to the test suite.
 Our tests are written for the `py.test`_ or nose_ test frameworks.
 At the point where you send your pull request, a command like this:
 
->>> py.test -n 4 --cov mitmproxy
+>>> py.test --cov mitmproxy --cov netlib
 
 Should give output something like this:
 

docs/index.rst

@@ -52,6 +52,7 @@
     :caption: Scripting
 
     scripting/inlinescripts
+    dev/models
     scripting/mitmproxy
 
 
@@ -73,7 +74,6 @@
     dev/protocols
     dev/proxy
     dev/exceptions
-    dev/models
 
 ..  Indices and tables
     ==================

docs/scripting/mitmproxy.rst

@@ -1,10 +1,10 @@
 
-mitmproxy
-=========
+FlowMaster
+==========
 
 .. note::
 
-    We strongly encourage you to use :ref:`inlinescripts` rather than mitmproxy.
+    We strongly encourage you to use :ref:`inlinescripts` rather than subclassing mitmproxy's FlowMaster.
         - Inline Scripts are equally powerful and provide an easier syntax.
         - Most examples are written as inline scripts.
         - Multiple inline scripts can be used together.

mitmproxy/models/connections.py

@@ -10,6 +10,17 @@ from .. import stateobject, utils
 
 
 class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
+    """
+    A client connection
+
+    Attributes:
+        address: Remote address
+        ssl_established: True if TLS is established, False otherwise
+        clientcert: The TLS client certificate
+        timestamp_start: Connection start timestamp
+        timestamp_ssl_setup: TLS established timestamp
+        timestamp_end: Connection end timestamp
+    """
     def __init__(self, client_connection, address, server):
         # Eventually, this object is restored from state. We don't have a
         # connection then.
@@ -47,11 +58,11 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
 
     _stateobject_attributes = dict(
         address=tcp.Address,
-        clientcert=certutils.SSLCert,
         ssl_established=bool,
+        clientcert=certutils.SSLCert,
         timestamp_start=float,
+        timestamp_ssl_setup=float,
         timestamp_end=float,
-        timestamp_ssl_setup=float
     )
 
     def copy(self):
@@ -90,6 +101,22 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
 
 
 class ServerConnection(tcp.TCPClient, stateobject.StateObject):
+    """
+    A server connection
+
+    Attributes:
+        address: Remote address. Can be both a domain or an IP address.
+        ip_address: Resolved remote IP address.
+        source_address: Local IP address
+        ssl_established: True if TLS is established, False otherwise
+        cert: The certificate presented by the remote during the TLS handshake
+        sni: Server Name Indication sent by the proxy during the TLS handshake
+        via: The underlying server connection (e.g. the connection to the upstream proxy in upstream proxy mode)
+        timestamp_start: Connection start timestamp
+        timestamp_tcp_setup: TCP ACK received timestamp
+        timestamp_ssl_setup: TLS established timestamp
+        timestamp_end: Connection end timestamp
+    """
     def __init__(self, address, source_address=None):
         tcp.TCPClient.__init__(self, address, source_address)
 
@@ -123,16 +150,16 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
         return self.ssl_established
 
     _stateobject_attributes = dict(
-        timestamp_start=float,
-        timestamp_end=float,
-        timestamp_tcp_setup=float,
-        timestamp_ssl_setup=float,
         address=tcp.Address,
         peer_address=tcp.Address,
         source_address=tcp.Address,
-        cert=certutils.SSLCert,
         ssl_established=bool,
-        sni=str
+        cert=certutils.SSLCert,
+        sni=str,
+        timestamp_start=float,
+        timestamp_tcp_setup=float,
+        timestamp_ssl_setup=float,
+        timestamp_end=float,
     )
 
     @classmethod

mitmproxy/models/http.py

@@ -30,41 +30,9 @@ class MessageMixin(object):
 class HTTPRequest(MessageMixin, Request):
 
     """
-    An HTTP request.
-
-    Exposes the following attributes:
-
-        method: HTTP method
-
-        scheme: URL scheme (http/https)
-
-        host: Target hostname of the request. This is not neccessarily the
-        directy upstream server (which could be another proxy), but it's always
-        the target server we want to reach at the end. This attribute is either
-        inferred from the request itself (absolute-form, authority-form) or from
-        the connection metadata (e.g. the host in reverse proxy mode).
-
-        port: Destination port
-
-        path: Path portion of the URL (not present in authority-form)
-
-        http_version: HTTP version, e.g. "HTTP/1.1"
-
-        headers: Headers object
-
-        content: Content of the request, the value is None if there is content
-        associated, but not present.
-
-        first_line_format: The request form. The following values are possible:
-
-             - relative (GET /index.html, OPTIONS *) (origin form or asterisk form)
-             - absolute (GET http://example.com:80/index.html)
-             - authority-form (CONNECT example.com:443)
-             Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3
-
-        timestamp_start: Timestamp indicating when request transmission started
-
-        timestamp_end: Timestamp indicating when request transmission ended
+    A mitmproxy HTTP request.
+    This is a very thin wrapper on top of :py:class:`netlib.http.Request` and
+    may be removed in the future.
     """
 
     def __init__(
@@ -123,6 +91,9 @@ class HTTPRequest(MessageMixin, Request):
 
     @classmethod
     def wrap(self, request):
+        """
+        Wraps an existing :py:class:`netlib.http.Request`.
+        """
         req = HTTPRequest(
             first_line_format=request.data.first_line_format,
             method=request.data.method,
@@ -141,27 +112,12 @@ class HTTPRequest(MessageMixin, Request):
     def __hash__(self):
         return id(self)
 
+
 class HTTPResponse(MessageMixin, Response):
-
     """
-    An HTTP response.
-
-    Exposes the following attributes:
-
-        http_version: HTTP version, e.g. "HTTP/1.1"
-
-        status_code: HTTP response status code
-
-        msg: HTTP response message
-
-        headers: Headers object
-
-        content: Content of the response, the value is None if there is content
-        associated, but not present.
-
-        timestamp_start: Timestamp indicating when request transmission started
-
-        timestamp_end: Timestamp indicating when request transmission ended
+    A mitmproxy HTTP response.
+    This is a very thin wrapper on top of :py:class:`netlib.http.Response` and
+    may be removed in the future.
     """
 
     def __init__(
@@ -192,6 +148,9 @@ class HTTPResponse(MessageMixin, Response):
 
     @classmethod
     def wrap(self, response):
+        """
+        Wraps an existing :py:class:`netlib.http.Response`.
+        """
         resp = HTTPResponse(
             http_version=response.data.http_version,
             status_code=response.data.status_code,
@@ -211,11 +170,11 @@ class HTTPFlow(Flow):
     transaction.
 
     Attributes:
-        request: HTTPRequest object
-        response: HTTPResponse object
-        error: Error object
-        server_conn: ServerConnection object
-        client_conn: ClientConnection object
+        request: :py:class:`HTTPRequest` object
+        response: :py:class:`HTTPResponse` object
+        error: :py:class:`Error` object
+        server_conn: :py:class:`ServerConnection` object
+        client_conn: :py:class:`ClientConnection` object
         intercepted: Is this flow currently being intercepted?
         live: Does this flow have a live client connection?