Merge pull request #34 from mitmproxy/netlib-http1-refactor

Adjust to netlib changes

.travis.yml

@@ -67,4 +67,4 @@ cache:
     - /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages
     - /home/travis/virtualenv/python2.7.9/bin
     - /home/travis/virtualenv/pypy-2.5.0/site-packages
-    - /home/travis/virtualenv/pypy-2.5.0/bin
+    - /home/travis/virtualenv/pypy-2.5.0/bin

libpathod/language/http2.py

@@ -1,6 +1,7 @@
 import pyparsing as pp
 
-from netlib.http import user_agents, semantics, Headers
+from netlib import http
+from netlib.http import user_agents, Headers
 from . import base, message
 
 """
@@ -184,7 +185,7 @@ class Response(_HTTP2Message):
             if body:
                 body = body.string()
 
-            resp = semantics.Response(
+            resp = http.Response(
                 (2, 0),
                 self.code.string(),
                 '',
@@ -267,7 +268,7 @@ class Request(_HTTP2Message):
             if body:
                 body = body.string()
 
-            req = semantics.Request(
+            req = http.Request(
                 '',
                 self.method.string(),
                 '',

libpathod/log.py

@@ -63,7 +63,7 @@ class LogCtx(object):
             for line in netlib.utils.hexdump(data):
                 self("\t%s %s %s" % line)
         else:
-            for i in netlib.utils.cleanBin(data).split("\n"):
+            for i in netlib.utils.clean_bin(data).split("\n"):
                 self("\t%s" % i)
 
     def __call__(self, line):

libpathod/pathoc.py

@@ -10,8 +10,10 @@ import time
 import threading
 
 import OpenSSL.crypto
+import six
 
 from netlib import tcp, http, certutils, websockets, socks
+from netlib.exceptions import HttpException
 from netlib.http import http1, http2
 
 import language.http
@@ -19,7 +21,7 @@ import language.websockets
 from . import utils, log
 
 import logging
-from netlib.http.http1 import HTTP1Protocol
+from netlib.tutils import treq
 
 logging.getLogger("hpack").setLevel(logging.WARNING)
 
@@ -213,7 +215,7 @@ class Pathoc(tcp.TCPClient):
                 )
             self.protocol = http2.HTTP2Protocol(self, dump_frames=self.http2_framedump)
         else:
-            self.protocol = http1.HTTP1Protocol(self)
+            self.protocol = http1
 
         self.settings = language.Settings(
             is_client=True,
@@ -229,15 +231,14 @@ class Pathoc(tcp.TCPClient):
             '\r\n'
         )
         self.wfile.flush()
-        l = self.rfile.readline()
+        try:
-        if not l:
+            resp = self.protocol.read_response(self.rfile, treq(method="CONNECT"))
-            raise PathocError("Proxy CONNECT failed")
+            if resp.status_code != 200:
-        parsed = self.protocol.parse_response_line(l)
+                raise HttpException("Unexpected status code: %s" % resp.status_code)
-        if not parsed[1] == 200:
+        except HttpException as e:
-            raise PathocError(
+            six.reraise(PathocError, PathocError(
-                "Proxy CONNECT failed: %s - %s" % (parsed[1], parsed[2])
+                "Proxy CONNECT failed: %s" % repr(e)
-            )
+            ))
-        self.protocol.read_headers()
 
     def socks_connect(self, connect_to):
         try:
@@ -288,9 +289,9 @@ class Pathoc(tcp.TCPClient):
         self.sslinfo = None
         if self.ssl:
             try:
-                alpn_protos = [HTTP1Protocol.ALPN_PROTO_HTTP1]
+                alpn_protos = [http.ALPN_PROTO_HTTP1]
                 if self.use_http2:
-                    alpn_protos.append(http2.HTTP2Protocol.ALPN_PROTO_H2)
+                    alpn_protos.append(http.ALPN_PROTO_H2)
 
                 self.convert_to_ssl(
                     sni=self.sni,
@@ -408,9 +409,9 @@ class Pathoc(tcp.TCPClient):
                 req = language.serve(r, self.wfile, self.settings)
                 self.wfile.flush()
 
-                resp = self.protocol.read_response(req["method"], None)
+                resp = self.protocol.read_response(self.rfile, treq(method=req["method"]))
                 resp.sslinfo = self.sslinfo
-            except http.HttpError as v:
+            except HttpException as v:
                 lg("Invalid server response: %s" % v)
                 raise
             except tcp.NetLibTimeout:

libpathod/pathod.py

@@ -6,7 +6,8 @@ import threading
 import urllib
 
 from netlib import tcp, http, certutils, websockets
-from netlib.http import http1, http2
+from netlib.exceptions import HttpException, HttpReadDisconnect
+from netlib.http import ALPN_PROTO_HTTP1, ALPN_PROTO_H2
 
 from . import version, app, language, utils, log, protocols
 import language.http
@@ -40,7 +41,7 @@ class SSLOptions(object):
         ssl_options=tcp.SSL_DEFAULT_OPTIONS,
         ciphers=None,
         certs=None,
-        alpn_select=http2.HTTP2Protocol.ALPN_PROTO_H2,
+        alpn_select=ALPN_PROTO_H2,
     ):
         self.confdir = confdir
         self.cn = cn
@@ -124,15 +125,14 @@ class PathodHandler(tcp.BaseHandler):
         """
         with logger.ctx() as lg:
             try:
-                req = self.protocol.read_request()
+                req = self.protocol.read_request(self.rfile)
-            except http.HttpError as s:
+            except HttpReadDisconnect:
+                return None, None
+            except HttpException as s:
                 s = str(s)
                 lg(s)
                 return None, dict(type="error", msg=s)
 
-            if isinstance(req, http.EmptyRequest):
-                return None, None
-
             if req.method == 'CONNECT':
                 return self.protocol.handle_http_connect([req.host, req.port, req.httpversion], lg)
 
@@ -259,7 +259,7 @@ class PathodHandler(tcp.BaseHandler):
                 return
 
             alp = self.get_alpn_proto_negotiated()
-            if alp == http2.HTTP2Protocol.ALPN_PROTO_H2:
+            if alp == ALPN_PROTO_H2:
                 self.protocol = protocols.http2.HTTP2Protocol(self)
                 self.use_http2 = True
 

libpathod/protocols/http.py

@@ -1,14 +1,12 @@
-from netlib import tcp, http, wsgi
+from netlib import tcp, wsgi
-from netlib.http import http1
+from netlib.exceptions import HttpReadDisconnect
-from .. import version, app, language, utils, log
+from netlib.http import http1, Request
+from .. import version, language
 
-class HTTPProtocol:
 
+class HTTPProtocol(object):
     def __init__(self, pathod_handler):
         self.pathod_handler = pathod_handler
-        self.wire_protocol = http1.HTTP1Protocol(
-            self.pathod_handler
-        )
 
     def make_error_response(self, reason, body):
         return language.http.make_error_response(reason, body)
@@ -70,4 +68,4 @@ class HTTPProtocol:
         return self.pathod_handler.handle_http_request, None
 
     def read_request(self, lg=None):
-        return self.wire_protocol.read_request(allow_empty=True)
+        return http1.read_request(self.pathod_handler.rfile)

libpathod/protocols/http2.py

@@ -14,7 +14,7 @@ class HTTP2Protocol:
 
     def read_request(self, lg=None):
         self.wire_protocol.perform_server_connection_preface()
-        return self.wire_protocol.read_request()
+        return self.wire_protocol.read_request(self.pathod_handler.rfile)
 
     def assemble(self, message):
         return self.wire_protocol.assemble(message)

test/test_pathoc.py

@@ -5,9 +5,11 @@ import OpenSSL
 from mock import Mock
 
 from netlib import tcp, http, socks
+from netlib.exceptions import HttpException
 from netlib.http import http1, http2
 
 from libpathod import pathoc, test, version, pathod, language
+from netlib.tutils import raises
 import tutils
 
 
@@ -82,7 +84,7 @@ class _TestDaemon:
                 r = r.freeze(language.Settings())
             try:
                 c.request(r)
-            except (http.HttpError, tcp.NetLibError):
+            except (HttpException, tcp.NetLibError):
                 pass
         return s.getvalue()
 
@@ -92,7 +94,7 @@ class TestDaemonSSL(_TestDaemon):
     ssloptions = pathod.SSLOptions(
         request_client_cert=True,
         sans=["test1.com", "test2.com"],
-        alpn_select=http2.HTTP2Protocol.ALPN_PROTO_H2,
+        alpn_select=http.ALPN_PROTO_H2,
     )
 
     def test_sni(self):
@@ -222,11 +224,13 @@ class TestDaemon(_TestDaemon):
         to = ("foobar", 80)
         c = pathoc.Pathoc(("127.0.0.1", self.d.port), fp=None)
         c.rfile, c.wfile = cStringIO.StringIO(), cStringIO.StringIO()
-        tutils.raises("connect failed", c.http_connect, to)
+        with raises("connect failed"):
+            c.http_connect(to)
         c.rfile = cStringIO.StringIO(
             "HTTP/1.1 500 OK\r\n"
         )
-        tutils.raises("connect failed", c.http_connect, to)
+        with raises("connect failed"):
+            c.http_connect(to)
         c.rfile = cStringIO.StringIO(
             "HTTP/1.1 200 OK\r\n"
         )
@@ -273,7 +277,7 @@ class TestDaemonHTTP2(_TestDaemon):
             c = pathoc.Pathoc(
                 ("127.0.0.1", self.d.port),
             )
-            assert isinstance(c.protocol, http1.HTTP1Protocol)
+            assert c.protocol == http1
 
         def test_http2_alpn(self):
             c = pathoc.Pathoc(

test/test_pathod.py

@@ -4,6 +4,7 @@ import OpenSSL
 
 from libpathod import pathod, version
 from netlib import tcp, http
+from netlib.exceptions import HttpException
 import tutils
 
 
@@ -180,16 +181,16 @@ class CommonTests(tutils.DaemonTests):
 
     def test_invalid_content_length(self):
         tutils.raises(
-            http.HttpError,
+            HttpException,
             self.pathoc,
             ["get:/:h'content-length'='foo'"]
         )
         l = self.d.last_log()
         assert l["type"] == "error"
-        assert "Content-Length unknown" in l["msg"]
+        assert "Unparseable Content Length" in l["msg"]
 
     def test_invalid_headers(self):
-        tutils.raises(http.HttpError, self.pathoc, ["get:/:h'\t'='foo'"])
+        tutils.raises(HttpException, self.pathoc, ["get:/:h'\t'='foo'"])
         l = self.d.last_log()
         assert l["type"] == "error"
         assert "Invalid headers" in l["msg"]
@@ -247,7 +248,7 @@ class TestDaemon(CommonTests):
 
     def test_connect_err(self):
         tutils.raises(
-            http.HttpError,
+            HttpException,
             self.pathoc,
             [r"get:'http://foo.com/p/202':da"],
             connect_to=("localhost", self.d.port)