get server reconnect right, fix timestamps

libmproxy/protocol.py

@@ -3,12 +3,10 @@ from netlib import http, http_status, tcp
 import netlib.utils
 from netlib.odict import ODictCaseless
 import select
-from proxy import ProxyError
+from proxy import ProxyError, KILL
 
-KILL = 0 # FIXME: Remove duplication with proxy module
 LEGACY = True
 
-
 def _handle(msg, conntype, connection_handler, *args, **kwargs):
     handler = None
     if conntype == "http":
@@ -106,11 +104,11 @@ class HTTPResponse(HTTPMessage):
         if not include_content:
             raise NotImplementedError
 
-        timestamp_start = libmproxy.utils.timestamp()
         httpversion, code, msg, headers, content = http.read_response(
             rfile,
             request_method,
             body_size_limit)
+        timestamp_start = rfile.first_byte_timestamp
         timestamp_end = libmproxy.utils.timestamp()
         return HTTPResponse(httpversion, code, msg, headers, content, timestamp_start, timestamp_end)
 
@@ -165,8 +163,8 @@ class HTTPRequest(HTTPMessage):
         httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \
             = None, None, None, None, None, None, None, None, None, None
 
-        timestamp_start = libmproxy.utils.timestamp()
         request_line = HTTPHandler.get_line(rfile)
+        timestamp_start = rfile.first_byte_timestamp
 
         request_line_parts = http.parse_init(request_line)
         if not request_line_parts:
@@ -210,33 +208,34 @@ class HTTPHandler(ProtocolHandler):
             pass
         self.c.close = True
 
-    """
+    def get_response_from_server(self, request):
-    def wait_for_message(self):
+        request_raw = request._assemble()
-        """
+
-        Check both the client connection and the server connection (if present) for readable data.
+        for i in range(2):
-        """
+            try:
-        conns = [self.c.client_conn.rfile]
+                self.c.server_conn.wfile.write(request_raw)
-        if self.c.server_conn:
+                self.c.server_conn.wfile.flush()
-            conns.append(self.c.server_conn.rfile)
+                return HTTPResponse.from_stream(self.c.server_conn.rfile, request.method,
-        while True:
+                                                body_size_limit=self.c.config.body_size_limit)
-            readable, _, _ = select.select(conns, [], [], 10)
+            except (tcp.NetLibDisconnect, http.HttpErrorConnClosed), v:
-            if self.c.client_conn.rfile in readable:
+                self.c.log("error in server communication: %s" % str(v))
-                return
+                if i < 1:
-            if self.c.server_conn.rfile in readable:
+                    # In any case, we try to reconnect at least once.
-                data = self.c.server_conn.rfile.read(1)
+                    # This is necessary because it might be possible that we already initiated an upstream connection
-                if data == "":
+                    # after clientconnect that has already been expired, e.g consider the following event log:
-                    raise tcp.NetLibDisconnect
+                    # > clientconnect (transparent mode destination known)
-                elif data == "\r" or data == "\n":
+                    # > serverconnect
-                    self.c.log("Received an empty line from server")
+                    # > read n% of large request
-                    pass  # Possible leftover from previous message
+                    # > server detects timeout, disconnects
+                    # > read (100-n)% of large request
+                    # > send large request upstream
+                    self.c.server_reconnect()
                 else:
-                    raise ProxyError(502, "Unexpected message from server")
+                    raise v
-    """
 
     def handle_flow(self):
         flow = HTTPFlow(self.c.client_conn, self.c.server_conn, None, None, None)
         try:
-            # self.wait_for_message()
             flow.request = HTTPRequest.from_stream(self.c.client_conn.rfile,
                                                    body_size_limit=self.c.config.body_size_limit)
             self.c.log("request", [flow.request._assemble_request_line(flow.request.form_in)])
@@ -250,11 +249,7 @@ class HTTPHandler(ProtocolHandler):
             if isinstance(request_reply, HTTPResponse):
                 flow.response = request_reply
             else:
-                raw = flow.request._assemble()
+                flow.response = self.get_response_from_server(flow.request)
-                self.c.server_conn.wfile.write(raw)
-                self.c.server_conn.wfile.flush()
-                flow.response = HTTPResponse.from_stream(self.c.server_conn.rfile, flow.request.method,
-                                                         body_size_limit=self.c.config.body_size_limit)
 
             self.c.log("response", [flow.response._assemble_response_line()])
             response_reply = self.c.channel.ask("response" if LEGACY else "httpresponse",

libmproxy/proxy.py

@@ -13,7 +13,7 @@ class ProxyError(Exception):
         self.code, self.msg, self.headers = code, msg, headers
 
     def __str__(self):
-        return "ProxyError(%s, %s)"%(self.code, self.msg)
+        return "ProxyError(%s, %s)" % (self.code, self.msg)
 
 
 import protocol
@@ -25,7 +25,8 @@ class Log:
 
 
 class ProxyConfig:
-    def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, forward_proxy=None, transparent_proxy=None, authenticator=None):
+    def __init__(self, certfile=None, cacert=None, clientcerts=None, no_upstream_cert=False, body_size_limit=None,
+                 reverse_proxy=None, forward_proxy=None, transparent_proxy=None, authenticator=None):
         self.certfile = certfile
         self.cacert = cacert
         self.clientcerts = clientcerts
@@ -91,7 +92,6 @@ class ServerConnection(tcp.TCPClient):
             raise ProxyError(400, str(v))
 
     def finish(self):
-        if self.connection:  # Eventually, we had an error during .connect() and aren't even connected.
         tcp.TCPClient.finish(self)
         self.timestamp_end = utils.timestamp()
 
@@ -141,9 +141,9 @@ class ConnectionHandler:
             self.mode = "transparent"
 
     def del_server_connection(self):
-        if self.server_conn:
+        if self.server_conn and self.server_conn.connection:  
             self.server_conn.finish()
-            self.log("serverdisconnect", ["%s:%s"%(self.server_conn.host, self.server_conn.port)])
+            self.log("serverdisconnect", ["%s:%s" % (self.server_conn.host, self.server_conn.port)])
             self.channel.tell("serverdisconnect", self)
         self.server_conn = None
         self.sni = None
@@ -161,10 +161,11 @@ class ConnectionHandler:
                 if self.config.reverse_proxy:
                     server_address = self.config.reverse_proxy[1:]
                 elif self.config.transparent_proxy:
-                    server_address = self.config.transparent_proxy["resolver"].original_addr(self.client_conn.connection)
+                    server_address = self.config.transparent_proxy["resolver"].original_addr(
+                        self.client_conn.connection)
                     if not server_address:
                         raise ProxyError(502, "Transparent mode failure: could not resolve original destination.")
-                    self.log("transparent to %s:%s"%server_address)
+                    self.log("transparent to %s:%s" % server_address)
 
             self.determine_conntype()
 
@@ -216,10 +217,10 @@ class ConnectionHandler:
             self.server_conn.connect()
         except tcp.NetLibError, v:
             raise ProxyError(502, v)
-        self.log("serverconnect", ["%s:%s"%(host, port)])
+        self.log("serverconnect", ["%s:%s" % (host, port)])
         self.channel.tell("serverconnect", self)
 
-    def establish_ssl(self, client, server):
+    def establish_ssl(self, client=False, server=False):
         """
         Establishes SSL on the existing connection(s) to the server or the client,
         as specified by the parameters. If the target server is on the pass-through list,
@@ -241,12 +242,20 @@ class ConnectionHandler:
             self.client_conn.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert,
                                             handle_sni=self.handle_sni)
 
+    def server_reconnect(self):
+        self.log("server reconnect")
+        had_ssl, sni = self.server_conn.ssl_established, self.sni
+        self.establish_server_connection(*self.server_conn.address)
+        if had_ssl:
+            self.sni = sni
+            self.establish_ssl(server=True)
+
     def log(self, msg, subs=()):
         msg = [
             "%s:%s: %s" % (self.client_conn.host, self.client_conn.port, msg)
         ]
         for i in subs:
-            msg.append("  -> "+i)
+            msg.append("  -> " + i)
         msg = "\n".join(msg)
         self.channel.tell("log", Log(msg))
 
@@ -291,12 +300,14 @@ class ConnectionHandler:
         except Exception, e: # pragma: no cover
             pass
 
+
 class ProxyServerError(Exception): pass
 
 
 class ProxyServer(tcp.TCPServer):
     allow_reuse_address = True
     bound = True
+
     def __init__(self, config, port, address='', server_version=version.NAMEVERSION):
         """
             Raises ProxyServerError if there's a startup problem.
@@ -324,6 +335,7 @@ class ProxyServer(tcp.TCPServer):
 
 class DummyServer:
     bound = False
+
     def __init__(self, config):
         self.config = config
 
@@ -339,22 +351,21 @@ def certificate_option_group(parser):
     group = parser.add_argument_group("SSL")
     group.add_argument(
         "--cert", action="store",
-        type = str, dest="cert", default=None,
+        type=str, dest="cert", default=None,
-        help = "User-created SSL certificate file."
+        help="User-created SSL certificate file."
     )
     group.add_argument(
         "--client-certs", action="store",
-        type = str, dest = "clientcerts", default=None,
+        type=str, dest="clientcerts", default=None,
-        help = "Client certificate directory."
+        help="Client certificate directory."
     )
 
 
-
 def process_proxy_options(parser, options):
     if options.cert:
         options.cert = os.path.expanduser(options.cert)
         if not os.path.exists(options.cert):
-            return parser.error("Manually created certificate does not exist: %s"%options.cert)
+            return parser.error("Manually created certificate does not exist: %s" % options.cert)
 
     cacert = os.path.join(options.confdir, "mitmproxy-ca.pem")
     cacert = os.path.expanduser(cacert)
@@ -368,8 +379,8 @@ def process_proxy_options(parser, options):
         if not platform.resolver:
             return parser.error("Transparent mode not supported on this platform.")
         trans = dict(
-            resolver = platform.resolver(),
+            resolver=platform.resolver(),
-            sslports = TRANSPARENT_SSL_PORTS
+            sslports=TRANSPARENT_SSL_PORTS
         )
     else:
         trans = None
@@ -377,14 +388,14 @@ def process_proxy_options(parser, options):
     if options.reverse_proxy:
         rp = utils.parse_proxy_spec(options.reverse_proxy)
         if not rp:
-            return parser.error("Invalid reverse proxy specification: %s"%options.reverse_proxy)
+            return parser.error("Invalid reverse proxy specification: %s" % options.reverse_proxy)
     else:
         rp = None
 
     if options.forward_proxy:
         fp = utils.parse_proxy_spec(options.forward_proxy)
         if not fp:
-            return parser.error("Invalid forward proxy specification: %s"%options.forward_proxy)
+            return parser.error("Invalid forward proxy specification: %s" % options.forward_proxy)
     else:
         fp = None
 
@@ -392,7 +403,7 @@ def process_proxy_options(parser, options):
         options.clientcerts = os.path.expanduser(options.clientcerts)
         if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts):
             return parser.error(
-                    "Client certificate directory does not exist or is not a directory: %s"%options.clientcerts
+                "Client certificate directory does not exist or is not a directory: %s" % options.clientcerts
             )
 
     if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd):
@@ -413,13 +424,13 @@ def process_proxy_options(parser, options):
         authenticator = http_auth.NullProxyAuth(None)
 
     return ProxyConfig(
-        certfile = options.cert,
+        certfile=options.cert,
-        cacert = cacert,
+        cacert=cacert,
-        clientcerts = options.clientcerts,
+        clientcerts=options.clientcerts,
-        body_size_limit = body_size_limit,
+        body_size_limit=body_size_limit,
-        no_upstream_cert = options.no_upstream_cert,
+        no_upstream_cert=options.no_upstream_cert,
-        reverse_proxy = rp,
+        reverse_proxy=rp,
-        forward_proxy = fp,
+        forward_proxy=fp,
-        transparent_proxy = trans,
+        transparent_proxy=trans,
-        authenticator = authenticator
+        authenticator=authenticator
     )