various fixes

libmproxy/filt.py

@@ -246,14 +246,14 @@ class FSrc(_Rex):
     help = "Match source address"
 
     def __call__(self, f):
-        return f.client_conn and re.search(self.expr, repr(f.client_conn.address))
+        return f.client_conn.address and re.search(self.expr, repr(f.client_conn.address))
 
 class FDst(_Rex):
     code = "dst"
     help = "Match destination address"
 
     def __call__(self, f):
-        return f.server_conn and re.search(self.expr, repr(f.server_conn.address))
+        return f.server_conn.address and re.search(self.expr, repr(f.server_conn.address))
 
 class _Int(_Action):
     def __init__(self, num):

libmproxy/protocol/http.py

@@ -729,6 +729,7 @@ class RequestReplayThread(threading.Thread):
             if not self.flow.response:
                 # In all modes, we directly connect to the server displayed
                 if self.config.mode == "upstream":
+                    # FIXME
                     server_address = self.config.mode.get_upstream_server(
                         self.flow.client_conn
                     )[2:]

libmproxy/protocol2/http.py

@@ -10,13 +10,14 @@ from libmproxy.protocol import KILL
 from libmproxy.protocol.http import HTTPFlow
 from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest
 from netlib import tcp
-from netlib.http import status_codes, http1, HttpErrorConnClosed
+from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError
 from netlib.http.semantics import CONTENT_MISSING
 from netlib import odict
 from netlib.tcp import NetLibError, Address
 from netlib.http.http1 import HTTP1Protocol
 from netlib.http.http2 import HTTP2Protocol
 
+
 # TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite.
 
 
@@ -31,6 +32,7 @@ class Http1Layer(Layer):
         layer = HttpLayer(self, self.mode)
         for message in layer():
             yield message
+            self.server_protocol = HTTP1Protocol(self.server_conn)
 
 
 class Http2Layer(Layer):
@@ -41,10 +43,10 @@ class Http2Layer(Layer):
         self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False)
 
     def __call__(self):
-        # FIXME: Handle Reconnect etc.
         layer = HttpLayer(self, self.mode)
         for message in layer():
             yield message
+            self.server_protocol = HTTP1Protocol(self.server_conn)
 
 
 def make_error_response(status_code, message, headers=None):
@@ -100,6 +102,7 @@ class ConnectServerConnection(object):
     """
     "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy.
     """
+
     def __init__(self, address, ctx):
         self.address = tcp.Address.wrap(address)
         self._ctx = ctx
@@ -124,6 +127,8 @@ class HttpLayer(Layer):
     def __call__(self):
         while True:
             try:
+                flow = HTTPFlow(self.client_conn, self.server_conn, live=True)
+
                 try:
                     request = HTTPRequest.from_protocol(
                         self.client_protocol,
@@ -148,7 +153,6 @@ class HttpLayer(Layer):
                 # Make sure that the incoming request matches our expectations
                 self.validate_request(request)
 
-                flow = HTTPFlow(self.client_conn, self.server_conn)
                 flow.request = request
                 for message in self.process_request_hook(flow):
                     yield message
@@ -164,15 +168,22 @@ class HttpLayer(Layer):
                 if self.check_close_connection(flow):
                     return
 
+                # TODO: Implement HTTP Upgrade
+
                 # Upstream Proxy Mode: Handle CONNECT
                 if flow.request.form_in == "authority" and flow.response.code == 200:
                     for message in self.handle_upstream_mode_connect(flow.request.copy()):
                         yield message
                     return
 
-            except (HttpErrorConnClosed, NetLibError) as e:
+            except (HttpErrorConnClosed, NetLibError, HttpError) as e:
-                make_error_response(502, repr(e))
+                self.send_to_client(make_error_response(
+                    getattr(e, "code", 502),
+                    repr(e)
+                ))
                 raise ProtocolException(repr(e), e)
+            finally:
+                flow.live = False
 
     def handle_regular_mode_connect(self, request):
         yield SetServer((request.host, request.port), False, None)
@@ -267,15 +278,16 @@ class HttpLayer(Layer):
 
             for chunk in chunks:
                 for part in chunk:
+                    # TODO: That's going to fail.
                     self.send_to_client(part)
                 self.client_conn.wfile.flush()
 
             flow.response.timestamp_end = utils.timestamp()
 
     def get_response_from_server(self, flow):
-        # TODO: Add second attempt.
+        def get_response():
             self.send_to_server(flow.request)
-
+            # Only get the headers at first...
             flow.response = HTTPResponse.from_protocol(
                 self.server_protocol,
                 flow.request.method,
@@ -283,6 +295,27 @@ class HttpLayer(Layer):
                 include_body=False,
             )
 
+        try:
+            get_response()
+        except (tcp.NetLibError, HttpErrorConnClosed) as v:
+            self.log(
+                "server communication error: %s" % repr(v),
+                level="debug"
+            )
+            # In any case, we try to reconnect at least once. This is
+            # necessary because it might be possible that we already
+            # initiated an upstream connection after clientconnect that
+            # has already been expired, e.g consider the following event
+            # log:
+            # > clientconnect (transparent mode destination known)
+            # > serverconnect (required for client tls handshake)
+            # > read n% of large request
+            # > server detects timeout, disconnects
+            # > read (100-n)% of large request
+            # > send large request upstream
+            yield Reconnect()
+            get_response()
+
         # call the appropriate script hook - this is an opportunity for an
         # inline script to set flow.stream = True
         flow = self.channel.ask("responseheaders", flow)
@@ -293,7 +326,6 @@ class HttpLayer(Layer):
             flow.response.content = CONTENT_MISSING
         else:
             flow.response.content = self.server_protocol.read_http_body(
-                self.server_conn,
                 flow.response.headers,
                 self.config.body_size_limit,
                 flow.request.method,
@@ -405,7 +437,7 @@ class HttpLayer(Layer):
                 self.send_to_client(make_error_response(
                     407,
                     "Proxy Authentication Required",
-                    self.config.authenticator.auth_challenge_headers()
+                    odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()])
                 ))
                 raise InvalidCredentials("Proxy Authentication Required")
 

libmproxy/protocol2/http_proxy.py

@@ -10,7 +10,8 @@ class HttpProxy(Layer, ServerConnectionMixin):
         for message in layer():
             if not self._handle_server_message(message):
                 yield message
-
+        if self.server_conn:
+            self._disconnect()
 
 class HttpUpstreamProxy(Layer, ServerConnectionMixin):
     def __init__(self, ctx, server_address):
@@ -21,3 +22,5 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin):
         for message in layer():
             if not self._handle_server_message(message):
                 yield message
+        if self.server_conn:
+            self._disconnect()

libmproxy/protocol2/layer.py

@@ -109,7 +109,9 @@ class ServerConnectionMixin(object):
 
     def _handle_server_message(self, message):
         if message == Reconnect:
+            address = self.server_conn.address
             self._disconnect()
+            self.server_conn.address = address
             self._connect()
             return True
         elif message == Connect:

libmproxy/protocol2/reverse_proxy.py

@@ -19,3 +19,5 @@ class ReverseProxy(Layer, ServerConnectionMixin):
         for message in layer():
             if not self._handle_server_message(message):
                 yield message
+        if self.server_conn:
+            self._disconnect()

libmproxy/protocol2/root_context.py

@@ -1,4 +1,5 @@
 from __future__ import (absolute_import, print_function, division)
+import string
 
 from .messages import Kill
 from .rawtcp import RawTcpLayer
@@ -36,6 +37,8 @@ class RootContext(object):
             d[2] in ('\x00', '\x01', '\x02', '\x03')
         )
 
+        is_ascii = all(x in string.ascii_uppercase for x in d)
+
         # TODO: build is_http2_magic check here, maybe this is an easy way to detect h2c
 
         if not d:
@@ -43,9 +46,10 @@ class RootContext(object):
 
         if is_tls_client_hello:
             return TlsLayer(top_layer, True, True)
-        elif isinstance(top_layer, TlsLayer) and top_layer.client_conn.get_alpn_proto_negotiated() == 'h2':
+        elif isinstance(top_layer, TlsLayer) and is_ascii:
+            if top_layer.client_conn.get_alpn_proto_negotiated() == 'h2':
                 return Http2Layer(top_layer, 'transparent')
-        elif isinstance(top_layer, TlsLayer) and isinstance(top_layer.ctx, Http1Layer):
+            else:
                 return Http1Layer(top_layer, "transparent")
         else:
             return RawTcpLayer(top_layer)

libmproxy/protocol2/socks_proxy.py

@@ -20,3 +20,5 @@ class Socks5Proxy(ServerConnectionMixin, Layer):
         for message in layer():
             if not self._handle_server_message(message):
                 yield message
+        if self.server_conn:
+            self._disconnect()

libmproxy/protocol2/transparent_proxy.py

@@ -21,3 +21,5 @@ class TransparentProxy(Layer, ServerConnectionMixin):
         for message in layer():
             if not self._handle_server_message(message):
                 yield message
+        if self.server_conn:
+            self._disconnect()

libmproxy/proxy/connection.py

@@ -27,6 +27,9 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         self.timestamp_ssl_setup = None
         self.protocol = None
 
+    def __nonzero__(self):
+        return bool(self.connection) and not self.finished
+
     def __repr__(self):
         return "<ClientConnection: {ssl}{host}:{port}>".format(
             ssl="[ssl] " if self.ssl_established else "",
@@ -89,7 +92,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
     def __init__(self, address):
         tcp.TCPClient.__init__(self, address)
 
-        self.state = []  # a list containing (conntype, state) tuples
+        self.via = None
         self.timestamp_start = None
         self.timestamp_end = None
         self.timestamp_tcp_setup = None
@@ -97,7 +100,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
         self.protocol = None
 
     def __nonzero__(self):
-        return bool(self.connection)
+        return bool(self.connection) and not self.finished
 
     def __repr__(self):
         if self.ssl_established and self.sni:
@@ -117,7 +120,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
         return self.ssl_established
 
     _stateobject_attributes = dict(
-        state=list,
         timestamp_start=float,
         timestamp_end=float,
         timestamp_tcp_setup=float,
@@ -187,3 +189,5 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
     def finish(self):
         tcp.TCPClient.finish(self)
         self.timestamp_end = utils.timestamp()
+
+ServerConnection._stateobject_attributes["via"] = ServerConnection

test/test_protocol_http.py

@@ -56,7 +56,7 @@ class TestInvalidRequests(tservers.HTTPProxTest):
         p = self.pathoc()
         r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port))
         assert r.status_code == 400
-        assert "Must not CONNECT on already encrypted connection" in r.body
+        assert "Invalid HTTP request form" in r.body
 
     def test_relative_request(self):
         p = self.pathoc_raw()