adjust to netlib changes

examples/har_extractor.py

@@ -128,7 +128,7 @@ def response(context, flow):
 
     request_query_string = [{"name": k, "value": v}
                             for k, v in flow.request.get_query()]
-    request_http_version = ".".join([str(v) for v in flow.request.httpversion])
+    request_http_version = flow.request.httpversion
     # Cookies are shaped as tuples by MITMProxy.
     request_cookies = [{"name": k.strip(), "value": v[0]}
                        for k, v in (flow.request.get_cookies() or {}).iteritems()]

libmproxy/cmdline.py

@@ -103,11 +103,15 @@ def parse_setheader(s):
 
 
 def parse_server_spec(url):
+    try:
         p = netlib.utils.parse_url(url)
-    if not p or not p[1] or p[0] not in ("http", "https"):
+        if p[0] not in ("http", "https"):
+            raise ValueError()
+    except ValueError:
         raise configargparse.ArgumentTypeError(
             "Invalid server specification: %s" % url
         )
+
     address = Address(p[1:3])
     scheme = p[0].lower()
     return config.ServerSpec(scheme, address)

libmproxy/console/common.py

@@ -4,7 +4,7 @@ import urwid
 import urwid.util
 import os
 
-from netlib.http.semantics import CONTENT_MISSING
+from netlib.http import CONTENT_MISSING
 import netlib.utils
 
 from .. import utils

libmproxy/console/flowview.py

@@ -6,7 +6,7 @@ import sys
 import urwid
 
 from netlib import odict
-from netlib.http.semantics import CONTENT_MISSING, Headers
+from netlib.http import CONTENT_MISSING, Headers
 from . import common, grideditor, signals, searchable, tabs
 from . import flowdetailview
 from .. import utils, controller, contentviews

libmproxy/dump.py

@@ -6,7 +6,7 @@ import traceback
 import click
 import itertools
 
-from netlib.http.semantics import CONTENT_MISSING
+from netlib.http import CONTENT_MISSING
 import netlib.utils
 from . import flow, filt, contentviews
 from .exceptions import ContentViewException

libmproxy/exceptions.py

@@ -34,11 +34,7 @@ class Socks5Exception(ProtocolException):
     pass
 
 
-class HttpException(ProtocolException):
+class HttpProtocolException(ProtocolException):
-    pass
-
-
-class InvalidCredentials(HttpException):
     pass
 
 
@@ -48,3 +44,7 @@ class ServerException(ProxyException):
 
 class ContentViewException(ProxyException):
     pass
+
+
+class ReplayException(ProxyException):
+    pass

libmproxy/flow.py

@@ -12,7 +12,8 @@ import urlparse
 
 
 from netlib import wsgi
-from netlib.http.semantics import CONTENT_MISSING, Headers
+from netlib.exceptions import HttpException
+from netlib.http import CONTENT_MISSING, Headers, http1
 import netlib.http
 from . import controller, tnetstring, filt, script, version
 from .onboarding import app
@@ -161,9 +162,8 @@ class StreamLargeBodies(object):
 
     def run(self, flow, is_request):
         r = flow.request if is_request else flow.response
-        code = flow.response.code if flow.response else None
+        expected_size = http1.expected_http_body_size(
-        expected_size = netlib.http.http1.HTTP1Protocol.expected_http_body_size(
+            flow.request, flow.response if not is_request else None
-            r.headers, is_request, flow.request.method, code
         )
         if not (0 <= expected_size <= self.max_size):
             # r.stream may already be a callable, which we want to preserve.
@@ -842,7 +842,7 @@ class FlowMaster(controller.Master):
             host,
             port,
             path,
-            (1, 1),
+            b"HTTP/1.1",
             headers,
             None,
             None,
@@ -1000,7 +1000,7 @@ class FlowMaster(controller.Master):
         try:
             if self.stream_large_bodies:
                 self.stream_large_bodies.run(f, False)
-        except netlib.http.HttpError:
+        except HttpException:
             f.reply(Kill)
             return
 

libmproxy/models/http.py

@@ -6,18 +6,17 @@ import time
 
 from libmproxy import utils
 from netlib import encoding
-from netlib.http import status_codes, Headers
+from netlib.http import status_codes, Headers, Request, Response, CONTENT_MISSING
 from netlib.tcp import Address
-from netlib.http.semantics import Request, Response, CONTENT_MISSING
 from .. import version, stateobject
 from .flow import Flow
 
 
 class MessageMixin(stateobject.StateObject):
     _stateobject_attributes = dict(
-        httpversion=tuple,
+        httpversion=bytes,
         headers=Headers,
-        body=str,
+        body=bytes,
         timestamp_start=float,
         timestamp_end=float
     )
@@ -120,7 +119,7 @@ class HTTPRequest(MessageMixin, Request):
 
         path: Path portion of the URL (not present in authority-form)
 
-        httpversion: HTTP version tuple, e.g. (1,1)
+        httpversion: HTTP version, e.g. "HTTP/1.1"
 
         headers: Headers object
 
@@ -186,11 +185,11 @@ class HTTPRequest(MessageMixin, Request):
     _stateobject_attributes = MessageMixin._stateobject_attributes.copy()
     _stateobject_attributes.update(
         form_in=str,
-        method=str,
+        method=bytes,
-        scheme=str,
+        scheme=bytes,
-        host=str,
+        host=bytes,
         port=int,
-        path=str,
+        path=bytes,
         form_out=str,
         is_replay=bool
     )
@@ -267,7 +266,7 @@ class HTTPResponse(MessageMixin, Response):
 
     Exposes the following attributes:
 
-        httpversion: HTTP version tuple, e.g. (1, 0), (1, 1), or (2, 0)
+        httpversion: HTTP version, e.g. "HTTP/1.1"
 
         status_code: HTTP response status code
 
@@ -312,7 +311,7 @@ class HTTPResponse(MessageMixin, Response):
     _stateobject_attributes = MessageMixin._stateobject_attributes.copy()
     _stateobject_attributes.update(
         status_code=int,
-        msg=str
+        msg=bytes
     )
 
     @classmethod
@@ -532,7 +531,7 @@ def make_error_response(status_code, message, headers=None):
         )
 
     return HTTPResponse(
-        (1, 1),  # FIXME: Should be a string.
+        b"HTTP/1.1",
         status_code,
         response,
         headers,
@@ -543,7 +542,7 @@ def make_error_response(status_code, message, headers=None):
 def make_connect_request(address):
     address = Address.wrap(address)
     return HTTPRequest(
-        "authority", "CONNECT", None, address.host, address.port, None, (1, 1),
+        "authority", "CONNECT", None, address.host, address.port, None, b"HTTP/1.1",
         Headers(), ""
     )
 

libmproxy/protocol/http.py

@@ -6,14 +6,14 @@ import traceback
 import six
 
 from netlib import tcp
-from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers
+from netlib.exceptions import HttpException, HttpReadDisconnect
-from netlib.http.semantics import CONTENT_MISSING
+from netlib.http import http1, Headers
+from netlib.http import CONTENT_MISSING
 from netlib.tcp import NetLibError, Address
-from netlib.http.http1 import HTTP1Protocol
+from netlib.http.http2.connections import HTTP2Protocol
-from netlib.http.http2 import HTTP2Protocol
 from netlib.http.http2.frame import GoAwayFrame, PriorityFrame, WindowUpdateFrame
 from .. import utils
-from ..exceptions import InvalidCredentials, HttpException, ProtocolException
+from ..exceptions import HttpProtocolException, ProtocolException
 from ..models import (
     HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error
 )
@@ -45,14 +45,14 @@ class _StreamingHttpLayer(_HttpLayer):
     def read_response_headers(self):
         raise NotImplementedError
 
-    def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
+    def read_response_body(self, request, response):
         raise NotImplementedError()
         yield "this is a generator"  # pragma: no cover
 
-    def read_response(self, request_method):
+    def read_response(self, request):
         response = self.read_response_headers()
-        response.body = "".join(
+        response.body = b"".join(
-            self.read_response_body(response.headers, request_method, response.code)
+            self.read_response_body(request, response)
         )
         return response
 
@@ -64,7 +64,7 @@ class _StreamingHttpLayer(_HttpLayer):
 
     def send_response(self, response):
         if response.body == CONTENT_MISSING:
-            raise HttpError(502, "Cannot assemble flow with CONTENT_MISSING")
+            raise HttpException("Cannot assemble flow with CONTENT_MISSING")
         self.send_response_headers(response)
         self.send_response_body(response, [response.body])
 
@@ -73,48 +73,31 @@ class Http1Layer(_StreamingHttpLayer):
     def __init__(self, ctx, mode):
         super(Http1Layer, self).__init__(ctx)
         self.mode = mode
-        self.client_protocol = HTTP1Protocol(self.client_conn)
-        self.server_protocol = HTTP1Protocol(self.server_conn)
 
     def read_request(self):
-        return HTTPRequest.from_protocol(
+        req = http1.read_request(self.client_conn.rfile, body_size_limit=self.config.body_size_limit)
-            self.client_protocol,
+        return HTTPRequest.wrap(req)
-            body_size_limit=self.config.body_size_limit
-        )
 
     def send_request(self, request):
-        self.server_conn.send(self.server_protocol.assemble(request))
+        self.server_conn.wfile.write(http1.assemble_request(request))
+        self.server_conn.wfile.flush()
 
     def read_response_headers(self):
-        return HTTPResponse.from_protocol(
+        resp = http1.read_response_head(self.server_conn.rfile)
-            self.server_protocol,
+        return HTTPResponse.wrap(resp)
-            request_method=None,  # does not matter if we don't read the body.
-            body_size_limit=self.config.body_size_limit,
-            include_body=False
-        )
 
-    def read_response_body(self, headers, request_method, response_code, max_chunk_size=None):
+    def read_response_body(self, request, response):
-        return self.server_protocol.read_http_body_chunked(
+        expected_size = http1.expected_http_body_size(request, response)
-            headers,
+        return http1.read_body(self.server_conn.rfile, expected_size, self.config.body_size_limit)
-            self.config.body_size_limit,
-            request_method,
-            response_code,
-            False,
-            max_chunk_size
-        )
 
     def send_response_headers(self, response):
-        h = self.client_protocol._assemble_response_first_line(response)
+        raw = http1.assemble_response_head(response, preserve_transfer_encoding=True)
-        self.client_conn.wfile.write(h + "\r\n")
+        self.client_conn.wfile.write(raw)
-        h = self.client_protocol._assemble_response_headers(
-            response,
-            preserve_transfer_encoding=True
-        )
-        self.client_conn.wfile.write(h + "\r\n")
         self.client_conn.wfile.flush()
 
     def send_response_body(self, response, chunks):
-        if self.client_protocol.has_chunked_encoding(response.headers):
+        if b"chunked" in response.headers.get(b"transfer-encoding", b"").lower():
+            # TODO: Move this into netlib.http.http1
             chunks = itertools.chain(
                 (
                     "{:x}\r\n{}\r\n".format(len(chunk), chunk)
@@ -127,36 +110,24 @@ class Http1Layer(_StreamingHttpLayer):
             self.client_conn.wfile.flush()
 
     def check_close_connection(self, flow):
-        close_connection = (
+        request_close = http1.connection_close(
-            http1.HTTP1Protocol.connection_close(
             flow.request.httpversion,
             flow.request.headers
-            ) or http1.HTTP1Protocol.connection_close(
+        )
+        response_close = http1.connection_close(
             flow.response.httpversion,
             flow.response.headers
-            ) or http1.HTTP1Protocol.expected_http_body_size(
-                flow.response.headers,
-                False,
-                flow.request.method,
-                flow.response.code) == -1
         )
+        read_until_eof = http1.expected_http_body_size(flow.request, flow.response) == -1
+        close_connection = request_close or response_close or read_until_eof
         if flow.request.form_in == "authority" and flow.response.code == 200:
-            # Workaround for
+            # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
-            # https://github.com/mitmproxy/mitmproxy/issues/313: Some
+            # Charles Proxy sends a CONNECT response with HTTP/1.0
-            # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0
             # and no Content-Length header
 
             return False
         return close_connection
 
-    def connect(self):
-        self.ctx.connect()
-        self.server_protocol = HTTP1Protocol(self.server_conn)
-
-    def set_server(self, *args, **kwargs):
-        self.ctx.set_server(*args, **kwargs)
-        self.server_protocol = HTTP1Protocol(self.server_conn)
-
     def __call__(self):
         layer = HttpLayer(self, self.mode)
         layer()
@@ -184,10 +155,10 @@ class Http2Layer(_HttpLayer):
         # TODO: implement flow control and WINDOW_UPDATE frames
         self.server_conn.send(self.server_protocol.assemble(message))
 
-    def read_response(self, request_method):
+    def read_response(self, request):
         return HTTPResponse.from_protocol(
             self.server_protocol,
-            request_method=request_method,
+            request_method=request.method,
             body_size_limit=self.config.body_size_limit,
             include_body=True,
             stream_id=self._stream_id
@@ -295,7 +266,7 @@ class UpstreamConnectLayer(Layer):
 
     def _send_connect_request(self):
         self.send_request(self.connect_request)
-        resp = self.read_response("CONNECT")
+        resp = self.read_response(self.connect_request)
         if resp.code != 200:
             raise ProtocolException("Reconnect: Upstream server refuses CONNECT request")
 
@@ -336,29 +307,32 @@ class HttpLayer(Layer):
         if self.mode == "transparent":
             self.__original_server_conn = self.server_conn
         while True:
-            try:
-                flow = HTTPFlow(self.client_conn, self.server_conn, live=self)
-
             try:
                 request = self.read_request()
-                except tcp.NetLibError:
-                    # don't throw an error for disconnects that happen
-                    # before/between requests.
-                    return
-
                 self.log("request", "debug", [repr(request)])
 
                 # Handle Proxy Authentication
-                self.authenticate(request)
+                if not self.authenticate(request):
+                    return
+
+                # Make sure that the incoming request matches our expectations
+                self.validate_request(request)
+
+            except HttpReadDisconnect:
+                # don't throw an error for disconnects that happen before/between requests.
+                return
+            except (HttpException, NetLibError) as e:
+                self.send_error_response(400, repr(e))
+                six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
+
+            try:
+                flow = HTTPFlow(self.client_conn, self.server_conn, live=self)
 
                 # Regular Proxy Mode: Handle CONNECT
                 if self.mode == "regular" and request.form_in == "authority":
                     self.handle_regular_mode_connect(request)
                     return
 
-                # Make sure that the incoming request matches our expectations
-                self.validate_request(request)
-
                 flow.request = request
                 self.process_request_hook(flow)
 
@@ -384,30 +358,26 @@ class HttpLayer(Layer):
                     self.handle_upstream_mode_connect(flow.request.copy())
                     return
 
-            except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e:
+            except (HttpException, NetLibError) as e:
-                error_propagated = False
+                self.send_error_response(502, repr(e))
-                if flow.request and not flow.response:
+
+                if not flow.response:
                     flow.error = Error(str(e))
                     self.channel.ask("error", flow)
                     self.log(traceback.format_exc(), "debug")
-                    error_propagated = True
+                    return
-
-                try:
-                    self.send_response(make_error_response(
-                        getattr(e, "code", 502),
-                        repr(e)
-                    ))
-                except NetLibError:
-                    pass
-
-                if not error_propagated:
-                    if isinstance(e, ProtocolException):
-                        six.reraise(ProtocolException, e, sys.exc_info()[2])
                 else:
                     six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
             finally:
                 flow.live = False
 
+    def send_error_response(self, code, message):
+        try:
+            response = make_error_response(code, message)
+            self.send_response(response)
+        except NetLibError:
+            pass
+
     def change_upstream_proxy_server(self, address):
         # Make set_upstream_proxy_server always available,
         # even if there's no UpstreamConnectLayer
@@ -435,10 +405,8 @@ class HttpLayer(Layer):
             # First send the headers and then transfer the response incrementally
             self.send_response_headers(flow.response)
             chunks = self.read_response_body(
-                flow.response.headers,
+                flow.request,
-                flow.request.method,
+                flow.response
-                flow.response.code,
-                max_chunk_size=4096
             )
             if callable(flow.response.stream):
                 chunks = flow.response.stream(chunks)
@@ -451,11 +419,11 @@ class HttpLayer(Layer):
             if self.supports_streaming:
                 flow.response = self.read_response_headers()
             else:
-                flow.response = self.read_response(flow.request.method)
+                flow.response = self.read_response(flow.request)
 
         try:
             get_response()
-        except (tcp.NetLibError, HttpErrorConnClosed) as v:
+        except (tcp.NetLibError, HttpException) as v:
             self.log(
                 "server communication error: %s" % repr(v),
                 level="debug"
@@ -485,10 +453,9 @@ class HttpLayer(Layer):
             if flow.response.stream:
                 flow.response.content = CONTENT_MISSING
             else:
-                flow.response.content = "".join(self.read_response_body(
+                flow.response.content = b"".join(self.read_response_body(
-                    flow.response.headers,
+                    flow.request,
-                    flow.request.method,
+                    flow.response
-                    flow.response.code
                 ))
             flow.response.timestamp_end = utils.timestamp()
 
@@ -543,7 +510,7 @@ class HttpLayer(Layer):
             if not self.server_conn:
                 self.connect()
             if tls:
-                raise HttpException("Cannot change scheme in upstream proxy mode.")
+                raise HttpProtocolException("Cannot change scheme in upstream proxy mode.")
             """
             # This is a very ugly (untested) workaround to solve a very ugly problem.
             if self.server_conn and self.server_conn.tls_established and not ssl:
@@ -561,12 +528,10 @@ class HttpLayer(Layer):
 
     def validate_request(self, request):
         if request.form_in == "absolute" and request.scheme != "http":
-            self.send_response(
-                make_error_response(400, "Invalid request scheme: %s" % request.scheme))
             raise HttpException("Invalid request scheme: %s" % request.scheme)
 
         expected_request_forms = {
-            "regular": ("absolute",),  # an authority request would already be handled.
+            "regular": ("authority", "absolute",),
             "upstream": ("authority", "absolute"),
             "transparent": ("relative",)
         }
@@ -576,10 +541,9 @@ class HttpLayer(Layer):
             err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
                 " or ".join(allowed_request_forms), request.form_in
             )
-            self.send_response(make_error_response(400, err_message))
             raise HttpException(err_message)
 
-        if self.mode == "regular":
+        if self.mode == "regular" and request.form_in == "absolute":
             request.form_out = "relative"
 
     def authenticate(self, request):
@@ -592,4 +556,5 @@ class HttpLayer(Layer):
                     "Proxy Authentication Required",
                     Headers(**self.config.authenticator.auth_challenge_headers())
                 ))
-                raise InvalidCredentials("Proxy Authentication Required")
+                return False
+        return True

libmproxy/protocol/http_replay.py

@@ -1,8 +1,9 @@
 from __future__ import (absolute_import, print_function, division)
 import threading
+from libmproxy.exceptions import ReplayException
+from netlib.exceptions import HttpException
+from netlib.http import http1
 
-from netlib.http import HttpError
-from netlib.http.http1 import HTTP1Protocol
 from netlib.tcp import NetLibError
 from ..controller import Channel
 from ..models import Error, HTTPResponse, ServerConnection, make_connect_request
@@ -47,13 +48,17 @@ class RequestReplayThread(threading.Thread):
                     server_address = self.config.upstream_server.address
                     server = ServerConnection(server_address)
                     server.connect()
-                    protocol = HTTP1Protocol(server)
                     if r.scheme == "https":
                         connect_request = make_connect_request((r.host, r.port))
-                        server.send(protocol.assemble(connect_request))
+                        server.wfile.write(http1.assemble_request(connect_request))
-                        resp = protocol.read_response("CONNECT")
+                        server.wfile.flush()
+                        resp = http1.read_response(
+                            server.rfile,
+                            connect_request,
+                            body_size_limit=self.config.body_size_limit
+                        )
                         if resp.code != 200:
-                            raise HttpError(502, "Upstream server refuses CONNECT request")
+                            raise ReplayException("Upstream server refuses CONNECT request")
                         server.establish_ssl(
                             self.config.clientcerts,
                             sni=self.flow.server_conn.sni
@@ -65,7 +70,6 @@ class RequestReplayThread(threading.Thread):
                     server_address = (r.host, r.port)
                     server = ServerConnection(server_address)
                     server.connect()
-                    protocol = HTTP1Protocol(server)
                     if r.scheme == "https":
                         server.establish_ssl(
                             self.config.clientcerts,
@@ -73,18 +77,19 @@ class RequestReplayThread(threading.Thread):
                         )
                     r.form_out = "relative"
 
-                server.send(protocol.assemble(r))
+                server.wfile.write(http1.assemble_request(r))
+                server.wfile.flush()
                 self.flow.server_conn = server
-                self.flow.response = HTTPResponse.from_protocol(
+                self.flow.response = http1.read_response(
-                    protocol,
+                    server.rfile,
-                    r.method,
+                    r,
-                    body_size_limit=self.config.body_size_limit,
+                    body_size_limit=self.config.body_size_limit
                 )
             if self.channel:
                 response_reply = self.channel.ask("response", self.flow)
                 if response_reply == Kill:
                     raise Kill()
-        except (HttpError, NetLibError) as v:
+        except (ReplayException, HttpException, NetLibError) as v:
             self.flow.error = Error(repr(v))
             if self.channel:
                 self.channel.ask("error", self.flow)

libmproxy/protocol/tls.py

@@ -7,7 +7,7 @@ from construct import ConstructError
 import six
 
 from netlib.tcp import NetLibError, NetLibInvalidCertificateError
-from netlib.http.http1 import HTTP1Protocol
+from netlib.http import ALPN_PROTO_HTTP1
 from ..contrib.tls._constructs import ClientHello
 from ..exceptions import ProtocolException, TlsException, ClientHandshakeException
 from .base import Layer
@@ -367,8 +367,8 @@ class TlsLayer(Layer):
         """
 
         # This gets triggered if we haven't established an upstream connection yet.
-        default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1
+        default_alpn = ALPN_PROTO_HTTP1
-        # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2
+        # alpn_preference = ALPN_PROTO_H2
 
         if self.alpn_for_client_connection in options:
             choice = bytes(self.alpn_for_client_connection)

libmproxy/proxy/root_context.py

@@ -5,8 +5,7 @@ import sys
 import six
 
 from libmproxy.exceptions import ProtocolException
-from netlib.http.http1 import HTTP1Protocol
+from netlib.http import ALPN_PROTO_H2, ALPN_PROTO_HTTP1
-from netlib.http.http2 import HTTP2Protocol
 from netlib.tcp import NetLibError
 from ..protocol import (
     RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin
@@ -85,9 +84,9 @@ class RootContext(object):
         # 5. Check for TLS ALPN (HTTP1/HTTP2)
         if isinstance(top_layer, TlsLayer):
             alpn = top_layer.client_conn.get_alpn_proto_negotiated()
-            if alpn == HTTP2Protocol.ALPN_PROTO_H2:
+            if alpn == ALPN_PROTO_H2:
                 return Http2Layer(top_layer, 'transparent')
-            if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1:
+            if alpn == ALPN_PROTO_HTTP1:
                 return Http1Layer(top_layer, 'transparent')
 
         # 6. Check for raw tcp mode

libmproxy/proxy/server.py

@@ -6,7 +6,7 @@ import socket
 import six
 
 from netlib import tcp
-from netlib.http.http1 import HTTP1Protocol
+from netlib.http.http1 import assemble_response
 from netlib.tcp import NetLibError
 from ..exceptions import ProtocolException, ServerException, ClientHandshakeException
 from ..protocol import Kill
@@ -138,7 +138,7 @@ class ConnectionHandler(object):
             # understandable by HTTP clients and humans.
             try:
                 error_response = make_error_response(502, repr(e))
-                self.client_conn.send(HTTP1Protocol().assemble(error_response))
+                self.client_conn.send(assemble_response(error_response))
             except NetLibError:
                 pass
         except Exception:

libmproxy/web/app.py

@@ -128,12 +128,10 @@ class FlowHandler(RequestHandler):
             if a == "request":
                 request = flow.request
                 for k, v in b.iteritems():
-                    if k in ["method", "scheme", "host", "path"]:
+                    if k in ["method", "scheme", "host", "path", "httpversion"]:
                         setattr(request, k, str(v))
                     elif k == "port":
                         request.port = int(v)
-                    elif k == "httpversion":
-                        request.httpversion = tuple(int(x) for x in v)
                     elif k == "headers":
                         request.headers.load_state(v)
                     else:
@@ -147,7 +145,7 @@ class FlowHandler(RequestHandler):
                     elif k == "code":
                         response.code = int(v)
                     elif k == "httpversion":
-                        response.httpversion = tuple(int(x) for x in v)
+                        response.httpversion = str(v)
                     elif k == "headers":
                         response.headers.load_state(v)
                     else:

test/test_dump.py

@@ -4,7 +4,7 @@ from libmproxy.exceptions import ContentViewException
 from libmproxy.models import HTTPResponse
 
 import netlib.tutils
-from netlib.http.semantics import CONTENT_MISSING
+from netlib.http import CONTENT_MISSING
 
 from libmproxy import dump, flow
 from libmproxy.proxy import Log
@@ -38,13 +38,13 @@ def test_strfuncs():
     flow.request.stickycookie = True
     flow.client_conn = mock.MagicMock()
     flow.client_conn.address.host = "foo"
-    flow.response = netlib.tutils.tresp(content=CONTENT_MISSING)
+    flow.response = netlib.tutils.tresp(body=CONTENT_MISSING)
     flow.response.is_replay = True
     flow.response.code = 300
     m.echo_flow(flow)
 
 
-    flow = tutils.tflow(resp=netlib.tutils.tresp("{"))
+    flow = tutils.tflow(resp=netlib.tutils.tresp(body="{"))
     flow.response.headers["content-type"] = "application/json"
     flow.response.code = 400
     m.echo_flow(flow)
@@ -62,14 +62,14 @@ def test_contentview(get_content_view):
 
 class TestDumpMaster:
     def _cycle(self, m, content):
-        f = tutils.tflow(req=netlib.tutils.treq(content))
+        f = tutils.tflow(req=netlib.tutils.treq(body=content))
         l = Log("connect")
         l.reply = mock.MagicMock()
         m.handle_log(l)
         m.handle_clientconnect(f.client_conn)
         m.handle_serverconnect(f.server_conn)
         m.handle_request(f)
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp(content))
+        f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=content))
         f = m.handle_response(f)
         m.handle_clientdisconnect(f.client_conn)
         return f

test/test_filt.py

@@ -84,7 +84,7 @@ class TestMatching:
             "host",
             80,
             "/path",
-            (1, 1),
+            b"HTTP/1.1",
             headers,
             "content_request",
             None,
@@ -99,8 +99,7 @@ class TestMatching:
 
         headers = Headers([["header_response", "svalue"]])
         f.response = http.HTTPResponse(
-            (1,
+            b"HTTP/1.1",
-             1),
             200,
             "OK",
             headers,

test/test_flow.py

@@ -8,7 +8,7 @@ import mock
 
 import netlib.utils
 from netlib import odict
-from netlib.http.semantics import CONTENT_MISSING, HDR_FORM_URLENCODED, Headers
+from netlib.http import CONTENT_MISSING, HDR_FORM_URLENCODED, Headers
 from libmproxy import filt, protocol, controller, tnetstring, flow
 from libmproxy.models import Error, Flow, HTTPRequest, HTTPResponse, HTTPFlow, decoded
 from libmproxy.proxy.config import HostMatcher
@@ -849,7 +849,7 @@ class TestFlowMaster:
         s = flow.State()
 
         f = tutils.tflow()
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp(f.request))
+        f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=f.request))
         pb = [f]
 
         fm = flow.FlowMaster(None, s)
@@ -903,7 +903,7 @@ class TestFlowMaster:
     def test_server_playback_kill(self):
         s = flow.State()
         f = tutils.tflow()
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp(f.request))
+        f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=f.request))
         pb = [f]
         fm = flow.FlowMaster(None, s)
         fm.refresh_server_playback = True
@@ -1043,7 +1043,7 @@ class TestRequest:
 
     def test_getset_form_urlencoded(self):
         d = odict.ODict([("one", "two"), ("three", "four")])
-        r = HTTPRequest.wrap(netlib.tutils.treq(content=netlib.utils.urlencode(d.lst)))
+        r = HTTPRequest.wrap(netlib.tutils.treq(body=netlib.utils.urlencode(d.lst)))
         r.headers["content-type"] = HDR_FORM_URLENCODED
         assert r.get_form_urlencoded() == d
 

test/test_fuzzing.py

@@ -17,15 +17,11 @@ class TestFuzzy(tservers.HTTPProxTest):
         p = self.pathoc()
         assert p.request(req % self.server.port).status_code == 400
 
-    def test_invalid_ports(self):
-        req = 'get:"http://localhost:999999"'
-        p = self.pathoc()
-        assert p.request(req).status_code == 400
-
     def test_invalid_ipv6_url(self):
         req = 'get:"http://localhost:%s":i13,"["'
         p = self.pathoc()
-        assert p.request(req % self.server.port).status_code == 400
+        resp = p.request(req % self.server.port)
+        assert resp.status_code == 400
 
     # def test_invalid_upstream(self):
     #     req = r"get:'http://localhost:%s/p/200:i10,\x27+\x27'"

test/test_protocol_http.py

@@ -1,46 +1,36 @@
-import cStringIO
+from io import BytesIO
-from cStringIO import StringIO
+from netlib.exceptions import HttpSyntaxException
 
-from mock import MagicMock
-
-from libmproxy.protocol.http import *
-import netlib.http
 from netlib.http import http1
-from netlib.http.semantics import CONTENT_MISSING
+from netlib.tutils import treq, raises
-
 import tutils
 import tservers
 
-def mock_protocol(data=''):
-    rfile = cStringIO.StringIO(data)
-    wfile = cStringIO.StringIO()
-    return http1.HTTP1Protocol(rfile=rfile, wfile=wfile)
-
 
 class TestHTTPResponse:
     def test_read_from_stringio(self):
-        s = "HTTP/1.1 200 OK\r\n" \
+        s = (
-             "Content-Length: 7\r\n" \
+            b"HTTP/1.1 200 OK\r\n"
-             "\r\n"\
+            b"Content-Length: 7\r\n"
-             "content\r\n" \
+            b"\r\n"
-             "HTTP/1.1 204 OK\r\n" \
+            b"content\r\n"
-             "\r\n"
+            b"HTTP/1.1 204 OK\r\n"
-
+            b"\r\n"
-        protocol = mock_protocol(s)
-        r = HTTPResponse.from_protocol(protocol, "GET")
-        assert r.status_code == 200
-        assert r.content == "content"
-        assert HTTPResponse.from_protocol(protocol, "GET").status_code == 204
-
-        protocol = mock_protocol(s)
-        # HEAD must not have content by spec. We should leave it on the pipe.
-        r = HTTPResponse.from_protocol(protocol, "HEAD")
-        assert r.status_code == 200
-        assert r.content == ""
-        tutils.raises(
-            "Invalid server response: 'content",
-            HTTPResponse.from_protocol, protocol, "GET"
         )
+        rfile = BytesIO(s)
+        r = http1.read_response(rfile, treq())
+        assert r.status_code == 200
+        assert r.content == b"content"
+        assert http1.read_response(rfile, treq()).status_code == 204
+
+        rfile = BytesIO(s)
+        # HEAD must not have content by spec. We should leave it on the pipe.
+        r = http1.read_response(rfile, treq(method=b"HEAD"))
+        assert r.status_code == 200
+        assert r.content == b""
+
+        with raises(HttpSyntaxException):
+            http1.read_response(rfile, treq())
 
 
 class TestHTTPFlow(object):

test/test_proxy.py

@@ -9,6 +9,7 @@ from libmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler
 import tutils
 from libpathod import test
 from netlib import http, tcp
+from netlib.http import http1
 
 
 class TestServerConnection:
@@ -26,11 +27,10 @@ class TestServerConnection:
         f.request.path = "/p/200:da"
 
         # use this protocol just to assemble - not for actual sending
-        protocol = http.http1.HTTP1Protocol(rfile=sc.rfile)
+        sc.wfile.write(http1.assemble_request(f.request))
-        sc.send(protocol.assemble(f.request))
+        sc.wfile.flush()
 
-        protocol = http.http1.HTTP1Protocol(rfile=sc.rfile)
+        assert http1.read_response(sc.rfile, f.request, 1000)
-        assert protocol.read_response(f.request.method, 1000)
         assert self.d.last_log()
 
         sc.finish()

test/test_server.py

@@ -1,13 +1,14 @@
 import socket
 import time
 from OpenSSL import SSL
+from netlib.exceptions import HttpReadDisconnect, HttpException
 from netlib.tcp import Address
 
 import netlib.tutils
 from netlib import tcp, http, socks
 from netlib.certutils import SSLCert
-from netlib.http import authentication
+from netlib.http import authentication, CONTENT_MISSING, http1
-from netlib.http.semantics import CONTENT_MISSING
+from netlib.tutils import raises
 from libpathod import pathoc, pathod
 
 from libmproxy.proxy.config import HostMatcher
@@ -143,10 +144,9 @@ class TcpMixin:
         # mitmproxy responds with bad gateway
         assert self.pathod(spec).status_code == 502
         self._ignore_on()
-        tutils.raises(
+        with raises(HttpException):
-            "invalid server response",
+            self.pathod(spec)  # pathoc tries to parse answer as HTTP
-            self.pathod,
+
-            spec)  # pathoc tries to parse answer as HTTP
         self._ignore_off()
 
     def _tcpproxy_on(self):
@@ -250,11 +250,6 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin):
         assert p.request(req % self.server2.urlbase)
         assert switched(self.proxy.log)
 
-    def test_get_connection_err(self):
-        p = self.pathoc()
-        ret = p.request("get:'http://localhost:0'")
-        assert ret.status_code == 502
-
     def test_blank_leading_line(self):
         p = self.pathoc()
         req = "get:'%s/p/201':i0,'\r\n'"
@@ -262,8 +257,8 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin):
 
     def test_invalid_headers(self):
         p = self.pathoc()
-        req = p.request("get:'http://foo':h':foo'='bar'")
+        resp = p.request("get:'http://foo':h':foo'='bar'")
-        assert req.status_code == 400
+        assert resp.status_code == 400
 
     def test_empty_chunked_content(self):
         """
@@ -570,17 +565,23 @@ class TestProxy(tservers.HTTPProxTest):
         connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         connection.connect(("localhost", self.proxy.port))
         connection.send(
-            "GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n" %
+            "GET http://localhost:%d/p/200:b@1k HTTP/1.1\r\n" %
             self.server.port)
         connection.send("\r\n")
-        connection.recv(5000)
+        # a bit hacky: make sure that we don't just read the headers only.
+        recvd = 0
+        while recvd < 1024:
+            recvd += len(connection.recv(5000))
         connection.send(
-            "GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n" %
+            "GET http://localhost:%d/p/200:b@1k HTTP/1.1\r\n" %
             self.server.port)
         connection.send("\r\n")
-        connection.recv(5000)
+        recvd = 0
+        while recvd < 1024:
+            recvd += len(connection.recv(5000))
         connection.close()
 
+        print(self.master.state.view._list)
         first_flow = self.master.state.view[0]
         second_flow = self.master.state.view[1]
         assert first_flow.server_conn.timestamp_tcp_setup
@@ -718,15 +719,12 @@ class TestStreamRequest(tservers.HTTPProxTest):
             (self.server.urlbase, spec))
         connection.send("\r\n")
 
-        protocol = http.http1.HTTP1Protocol(rfile=fconn)
+        resp = http1.read_response_head(fconn)
-        resp = protocol.read_response("GET", None, include_body=False)
 
         assert resp.headers["Transfer-Encoding"] == 'chunked'
         assert resp.status_code == 200
 
-        chunks = list(protocol.read_http_body_chunked(
+        chunks = list(http1.read_body(fconn, None))
-            resp.headers, None, "GET", 200, False
-        ))
         assert chunks == ["this", "isatest__reachhex"]
 
         connection.close()
@@ -743,7 +741,7 @@ class TestFakeResponse(tservers.HTTPProxTest):
 
     def test_fake(self):
         f = self.pathod("200")
-        assert "header_response" in f.headers
+        assert "header-response" in f.headers
 
 
 class TestServerConnect(tservers.HTTPProxTest):
@@ -766,7 +764,8 @@ class TestKillRequest(tservers.HTTPProxTest):
     masterclass = MasterKillRequest
 
     def test_kill(self):
-        tutils.raises("server disconnect", self.pathod, "200")
+        with raises(HttpReadDisconnect):
+            self.pathod("200")
         # Nothing should have hit the server
         assert not self.server.last_log()
 
@@ -780,7 +779,8 @@ class TestKillResponse(tservers.HTTPProxTest):
     masterclass = MasterKillResponse
 
     def test_kill(self):
-        tutils.raises("server disconnect", self.pathod, "200")
+        with raises(HttpReadDisconnect):
+            self.pathod("200")
         # The server should have seen a request
         assert self.server.last_log()
 
@@ -907,7 +907,7 @@ class TestUpstreamProxySSL(
         """
 
         def handle_request(f):
-            f.request.httpversion = (1, 0)
+            f.request.httpversion = b"HTTP/1.1"
             del f.request.headers["Content-Length"]
             f.reply()
 

test/tutils.py

@@ -102,7 +102,7 @@ def tflowview(request_contents=None):
     if request_contents is None:
         flow = tflow()
     else:
-        flow = tflow(req=netlib.tutils.treq(request_contents))
+        flow = tflow(req=netlib.tutils.treq(body=request_contents))
 
     fv = FlowView(m, cs, flow)
     return fv

web/src/js/components/flowview/messages.js

@@ -121,7 +121,7 @@ var RequestLine = React.createClass({
     render: function () {
         var flow = this.props.flow;
         var url = flowutils.RequestUtils.pretty_url(flow.request);
-        var httpver = "HTTP/" + flow.request.httpversion.join(".");
+        var httpver = flow.request.httpversion;
 
         return <div className="first-line request-line">
             <ValueEditor
@@ -175,7 +175,7 @@ var RequestLine = React.createClass({
 var ResponseLine = React.createClass({
     render: function () {
         var flow = this.props.flow;
-        var httpver = "HTTP/" + flow.response.httpversion.join(".");
+        var httpver = flow.response.httpversion;
         return <div className="first-line response-line">
             <ValueEditor
                 ref="httpVersion"