http2: add request-response to pathod

libpathod/language/__init__.py

@@ -19,7 +19,7 @@ def expand(msg):
         yield msg
 
 
-def parse_pathod(s):
+def parse_pathod(s, use_http2=False):
     """
         May raise ParseException
     """
@@ -28,12 +28,17 @@ def parse_pathod(s):
     except UnicodeError:
         raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0)
     try:
-        reqs = pp.Or(
+        if use_http2:
-            [
+            expressions = [
+                # http2.Frame.expr(),
+                http2.Response.expr(),
+            ]
+        else:
+            expressions = [
                 websockets.WebsocketFrame.expr(),
                 http.Response.expr(),
             ]
-        ).parseString(s, parseAll=True)
+        reqs = pp.Or(expressions).parseString(s, parseAll=True)
     except pp.ParseException as v:
         raise exceptions.ParseException(v.msg, v.line, v.col)
     return itertools.chain(*[expand(i) for i in reqs])
@@ -55,7 +60,6 @@ def parse_pathoc(s, use_http2=False):
                 websockets.WebsocketClientFrame.expr(),
                 http.Request.expr(),
             ]
-
         reqs = pp.OneOrMore(pp.Or(expressions)).parseString(s, parseAll=True)
     except pp.ParseException as v:
         raise exceptions.ParseException(v.msg, v.line, v.col)

libpathod/language/http.py

@@ -367,10 +367,6 @@ class Request(_HTTPMessage):
         return ":".join([i.spec() for i in self.tokens])
 
 
-class PathodErrorResponse(Response):
-    pass
-
-
 def make_error_response(reason, body=None):
     tokens = [
         Code("800"),
@@ -381,4 +377,4 @@ def make_error_response(reason, body=None):
         Reason(base.TokValueLiteral(reason)),
         Body(base.TokValueLiteral("pathod error: " + (body or reason))),
     ]
-    return PathodErrorResponse(tokens)
+    return Response(tokens)

libpathod/language/http2.py

@@ -45,14 +45,20 @@ class Body(base.Value):
 class Times(base.Integer):
     preamble = "x"
 
+class Code(base.Integer):
+    pass
 
 class Request(message.Message):
     comps = (
         Header,
         Body,
-
         Times,
     )
+    logattrs = ["method", "path"]
+
+    def __init__(self, tokens):
+        super(Response, self).__init__(tokens)
+        self.rendered_values = None
 
     @property
     def method(self):
@@ -87,7 +93,6 @@ class Request(message.Message):
                 Method.expr(),
                 base.Sep,
                 Path.expr(),
-                base.Sep,
                 pp.ZeroOrMore(base.Sep + atom)
             ]
         )
@@ -95,21 +100,109 @@ class Request(message.Message):
         return resp
 
     def resolve(self, settings, msg=None):
-        tokens = self.tokens[:]
+        return self
-        return self.__class__(
-            [i.resolve(settings, self) for i in tokens]
-        )
 
     def values(self, settings):
-        return settings.protocol.create_request(
+        if self.rendered_values:
+            return self.rendered_values
+        else:
+            headers = self.headers
+            if headers:
+                headers = headers.values(settings)
+
+            body = self.body
+            if body:
+                body = body.string()
+
+            self.rendered_values = settings.protocol.create_request(
                 self.method.string(),
-            self.path,
+                self.path.string(),
-            self.headers,
+                headers,  # TODO: parse that into a dict?!
-            self.body)
+                body)
+            return self.rendered_values
 
     def spec(self):
         return ":".join([i.spec() for i in self.tokens])
 
 
+class Response(message.Message):
+    unique_name = None
+    comps = (
+        Header,
+        Body,
+    )
+
+    def __init__(self, tokens):
+        super(Response, self).__init__(tokens)
+        self.rendered_values = None
+        self.stream_id = 0
+
+    @property
+    def code(self):
+        return self.tok(Code)
+
+    @property
+    def headers(self):
+        return self.toks(Header)
+
+    @property
+    def body(self):
+        return self.tok(Body)
+
+    @property
+    def actions(self):
+        return []
+
+    def resolve(self, settings, msg=None):
+        return self
+
+    @classmethod
+    def expr(klass):
+        parts = [i.expr() for i in klass.comps]
+        atom = pp.MatchFirst(parts)
+        resp = pp.And(
+            [
+                Code.expr(),
+                pp.ZeroOrMore(base.Sep + atom)
+            ]
+        )
+        resp = resp.setParseAction(klass)
+        return resp
+
+    def values(self, settings):
+        if self.rendered_values:
+            return self.rendered_values
+        else:
+            headers = self.headers
+            if headers:
+                headers = headers.values(settings)
+
+            body = self.body
+            if body:
+                body = body.values(settings)
+
+            self.rendered_values = settings.protocol.create_response(
+                self.code.string(),
+                self.stream_id,
+                headers,  # TODO: parse that into a dict?!
+                body)
+            return self.rendered_values
+
+    def spec(self):
+        return ":".join([i.spec() for i in self.tokens])
+
+def make_error_response(reason, body=None):
+    raise NotImplementedError
+    # tokens = [
+    #     Code("800"),
+    #     Header(
+    #         base.TokValueLiteral("Content-Type"),
+    #         base.TokValueLiteral("text/plain")
+    #     ),
+    #     Reason(base.TokValueLiteral(reason)),
+    #     Body(base.TokValueLiteral("pathod error: " + (body or reason))),
+    # ]
+    # return Response(tokens)
+
 # class Frame(message.Message):
 #     pass

libpathod/pathoc.py

@@ -155,13 +155,14 @@ class Pathoc(tcp.TCPClient):
             # SSL
             ssl=None,
             sni=None,
-            sslversion=4,
+            sslversion='SSLv23',
             clientcert=None,
             ciphers=None,
 
             # HTTP/2
             use_http2=False,
             http2_skip_connection_preface=False,
+            http2_framedump = False,
 
             # Websockets
             ws_read_limit = None,
@@ -199,6 +200,7 @@ class Pathoc(tcp.TCPClient):
 
         self.use_http2 = use_http2
         self.http2_skip_connection_preface = http2_skip_connection_preface
+        self.http2_framedump = http2_framedump
 
         self.ws_read_limit = ws_read_limit
 
@@ -219,7 +221,6 @@ class Pathoc(tcp.TCPClient):
             if not OpenSSL._util.lib.Cryptography_HAS_ALPN:  # pragma: nocover
                 print >> sys.stderr, "HTTP/2 requires ALPN support. Please use OpenSSL >= 1.0.2."
                 print >> sys.stderr, "Pathoc might not be working as expected without ALPN."
-
             self.protocol = http2.HTTP2Protocol(self)
         else:
             # TODO: create HTTP or Websockets protocol
@@ -298,7 +299,7 @@ class Pathoc(tcp.TCPClient):
             if self.use_http2:
                 self.protocol.check_alpn()
                 if not self.http2_skip_connection_preface:
-                    self.protocol.perform_connection_preface()
+                    self.protocol.perform_client_connection_preface()
 
         if self.timeout:
             self.settimeout(self.timeout)
@@ -466,6 +467,7 @@ def main(args):  # pragma: nocover
                 ciphers = args.ciphers,
                 use_http2 = args.use_http2,
                 http2_skip_connection_preface = args.http2_skip_connection_preface,
+                http2_framedump = args.http2_framedump,
                 showreq = args.showreq,
                 showresp = args.showresp,
                 explain = args.explain,

libpathod/pathoc_cmdline.py

@@ -109,12 +109,11 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
         help="SSL cipher specification"
     )
     group.add_argument(
-        "--sslversion", dest="sslversion", type=int, default=4,
+        "--sslversion", dest="sslversion", type=str, default='SSLv23',
-        choices=[1, 2, 3, 4],
+        choices=utils.SSLVERSIONS.keys(),
-        help="""
+        help=""""
-            Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
+            Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23.
-            to SSLv23.
+            Default to SSLv23."""
-        """
     )
 
     group = parser.add_argument_group(
@@ -156,6 +155,10 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
         "-x", dest="hexdump", action="store_true", default=False,
         help="Output in hexdump format"
     )
+    group.add_argument(
+        "--http2-framedump", dest="http2_framedump", action="store_true", default=False,
+        help="Output all received & sent HTTP/2 frames"
+    )
 
     args = parser.parse_args(argv[1:])
 

libpathod/pathod.py

@@ -7,7 +7,7 @@ import urllib
 import re
 import time
 
-from netlib import tcp, http, wsgi, certutils, websockets
+from netlib import tcp, http, http2, wsgi, certutils, websockets, odict
 
 from . import version, app, language, utils, log
 import language.http
@@ -40,7 +40,7 @@ class SSLOptions:
         sslversion=tcp.SSLv23_METHOD,
         ciphers=None,
         certs=None,
-        alpn_select=None,
+        alpn_select=http2.HTTP2Protocol.ALPN_PROTO_H2,
     ):
         self.confdir = confdir
         self.cn = cn
@@ -69,32 +69,37 @@ class PathodHandler(tcp.BaseHandler):
     wbufsize = 0
     sni = None
 
-    def __init__(self, connection, address, server, logfp, settings):
+    def __init__(self, connection, address, server, logfp, settings, http2_framedump=False):
-        self.logfp = logfp
         tcp.BaseHandler.__init__(self, connection, address, server)
+        self.logfp = logfp
         self.settings = copy.copy(settings)
+        self.protocol = None
+        self.use_http2 = False
+        self.http2_framedump = http2_framedump
 
-    def handle_sni(self, connection):
+    def _handle_sni(self, connection):
         self.sni = connection.get_servername()
 
     def http_serve_crafted(self, crafted):
+        """
+            This method is HTTP/1 and HTTP/2 capable.
+        """
+
         error, crafted = self.server.check_policy(
             crafted, self.settings
         )
         if error:
-            err = language.http.make_error_response(error)
+            err = self.make_http_error_response(error)
             language.serve(err, self.wfile, self.settings)
             return None, dict(
                 type="error",
                 msg = error
             )
 
-        if self.server.explain and not isinstance(
+        if self.server.explain and not hasattr(crafted, 'is_error_response'):
-                crafted,
-                language.http.PathodErrorResponse
-        ):
             crafted = crafted.freeze(self.settings)
             log.write(self.logfp, ">> Spec: %s" % crafted.spec())
+
         response_log = language.serve(
             crafted,
             self.wfile,
@@ -154,6 +159,8 @@ class PathodHandler(tcp.BaseHandler):
 
     def handle_http_connect(self, connect, lg):
         """
+            This method is HTTP/1 only.
+
             Handle a CONNECT request.
         """
         http.read_headers(self.rfile)
@@ -171,7 +178,7 @@ class PathodHandler(tcp.BaseHandler):
                 self.convert_to_ssl(
                     cert,
                     key,
-                    handle_sni=self.handle_sni,
+                    handle_sni=self._handle_sni,
                     request_client_cert=self.server.ssloptions.request_client_cert,
                     cipher_list=self.server.ssloptions.ciphers,
                     method=self.server.ssloptions.sslversion,
@@ -185,10 +192,12 @@ class PathodHandler(tcp.BaseHandler):
 
     def handle_http_app(self, method, path, headers, content, lg):
         """
+            This method is HTTP/1 only.
+
             Handle a request to the built-in app.
         """
         if self.server.noweb:
-            crafted = language.http.make_error_response("Access Denied")
+            crafted = self.make_http_error_response("Access Denied")
             language.serve(crafted, self.wfile, self.settings)
             return None, dict(
                 type="error",
@@ -209,6 +218,8 @@ class PathodHandler(tcp.BaseHandler):
 
     def handle_http_request(self):
         """
+            This method is HTTP/1 and HTTP/2 capable.
+
             Returns a (handler, log) tuple.
 
             handler: Handler for the next request, or None to disconnect
@@ -217,28 +228,24 @@ class PathodHandler(tcp.BaseHandler):
         lr = self.rfile if self.server.logreq else None
         lw = self.wfile if self.server.logresp else None
         with log.Log(self.logfp, self.server.hexdump, lr, lw) as lg:
-            line = http.get_request_line(self.rfile)
+            if self.use_http2:
-            if not line:
+                self.protocol.perform_server_connection_preface()
-                # Normal termination
+                stream_id, headers, body = self.protocol.read_request()
-                return None, None
+                method = headers[':method']
-
+                path = headers[':path']
-            m = utils.MemBool()
+                headers = odict.ODict(headers)
-            if m(http.parse_init_connect(line)):
-                return self.handle_http_connect(m.v, lg)
-            elif m(http.parse_init_proxy(line)):
-                method, _, _, _, path, httpversion = m.v
-            elif m(http.parse_init_http(line)):
-                method, path, httpversion = m.v
             else:
-                s = "Invalid first line: %s" % repr(line)
+                req = self.read_http_request(lg)
-                lg(s)
+                if 'next_handle' in req:
-                return None, dict(type="error", msg=s)
+                    return req['next_handle']
-
+                if 'errors' in req:
-            headers = http.read_headers(self.rfile)
+                    return None, req['errors']
-            if headers is None:
+                if not 'method' in req or not 'path' in req:
-                s = "Invalid headers"
+                    return None, None
-                lg(s)
+                method = req['method']
-                return None, dict(type="error", msg=s)
+                path = req['path']
+                headers = req['headers']
+                body = req['body']
 
             clientcert = None
             if self.clientcert:
@@ -258,7 +265,7 @@ class PathodHandler(tcp.BaseHandler):
                     path=path,
                     method=method,
                     headers=headers.lst,
-                    httpversion=httpversion,
+                    # httpversion=httpversion,
                     sni=self.sni,
                     remote_address=self.address(),
                     clientcert=clientcert,
@@ -268,16 +275,6 @@ class PathodHandler(tcp.BaseHandler):
             if self.ssl_established:
                 retlog["cipher"] = self.get_current_cipher()
 
-            try:
-                content = http.read_http_body(
-                    self.rfile, headers, None,
-                    method, None, True
-                )
-            except http.HttpError as s:
-                s = str(s)
-                lg(s)
-                return None, dict(type="error", msg=s)
-
             m = utils.MemBool()
             websocket_key = websockets.check_client_handshake(headers)
             self.settings.websocket_key = websocket_key
@@ -288,28 +285,37 @@ class PathodHandler(tcp.BaseHandler):
                 anchor_gen = language.parse_pathod("ws")
             else:
                 anchor_gen = None
-            for i in self.server.anchors:
+
-                if i[0].match(path):
+            for regex, spec in self.server.anchors:
-                    anchor_gen = i[1]
+                if regex.match(path):
+                    anchor_gen = language.parse_pathod(spec, self.use_http2)
                     break
             else:
-                print(self.server.craftanchor)
                 if m(path.startswith(self.server.craftanchor)):
                     spec = urllib.unquote(path)[len(self.server.craftanchor):]
                     if spec:
                         try:
-                            anchor_gen = language.parse_pathod(spec)
+                            anchor_gen = language.parse_pathod(spec, self.use_http2)
                         except language.ParseException as v:
                             lg("Parse error: %s" % v.msg)
-                            anchor_gen = iter([language.http.make_error_response(
+                            anchor_gen = iter([self.make_http_error_response(
                                 "Parse Error",
                                 "Error parsing response spec: %s\n" % (
                                     v.msg + v.marked()
                                 )
                             )])
+                else:
+                    if self.use_http2:
+                        raise NotImplementedError(\
+                            "HTTP/2 only supports request/response with the craft anchor point.")
+
 
             if anchor_gen:
                 spec = anchor_gen.next()
+
+                if self.use_http2 and isinstance(spec, language.http2.Response):
+                    spec.stream_id = stream_id
+
                 lg("crafting spec: %s" % spec)
                 nexthandler, retlog["response"] = self.http_serve_crafted(
                     spec
@@ -319,28 +325,77 @@ class PathodHandler(tcp.BaseHandler):
                 else:
                     return nexthandler, retlog
             else:
-                return self.handle_http_app(method, path, headers, content, lg)
+                return self.handle_http_app(method, path, headers, body, lg)
 
-    def addlog(self, log):
+    def read_http_request(self, lg):
-        # FIXME: The bytes in the log should not be escaped. We do this at the
+        """
-        # moment because JSON encoding can't handle binary data, and I don't
+            This method is HTTP/1 only.
-        # want to base64 everything.
+        """
-        if self.server.logreq:
+        line = http.get_request_line(self.rfile)
-            bytes = self.rfile.get_log().encode("string_escape")
+        if not line:
-            log["request_bytes"] = bytes
+            # Normal termination
-        if self.server.logresp:
+            return dict()
-            bytes = self.wfile.get_log().encode("string_escape")
+
-            log["response_bytes"] = bytes
+        m = utils.MemBool()
-        self.server.add_log(log)
+        if m(http.parse_init_connect(line)):
+            return dict(next_handle=self.handle_http_connect(m.v, lg))
+        elif m(http.parse_init_proxy(line)):
+            method, _, _, _, path, httpversion = m.v
+        elif m(http.parse_init_http(line)):
+            method, path, httpversion = m.v
+        else:
+            s = "Invalid first line: %s" % repr(line)
+            lg(s)
+            return dict(errors=dict(type="error", msg=s))
+
+        headers = http.read_headers(self.rfile)
+        if headers is None:
+            s = "Invalid headers"
+            lg(s)
+            return dict(errors=dict(type="error", msg=s))
+
+        try:
+            body = http.read_http_body(
+                self.rfile,
+                headers,
+                None,
+                method,
+                None,
+                True,
+            )
+        except http.HttpError as s:
+            s = str(s)
+            lg(s)
+            return dict(errors=dict(type="error", msg=s))
+
+        return dict(
+            method=method,
+            path=path,
+            headers=headers,
+            body=body)
+
+    def make_http_error_response(self, reason, body=None):
+        """
+            This method is HTTP/1 and HTTP/2 capable.
+        """
+        if self.use_http2:
+            resp = language.http2.make_error_response(reason, body)
+        else:
+            resp = language.http.make_error_response(reason, body)
+        resp.is_error_response = True
+        return resp
 
     def handle(self):
+        self.settimeout(self.server.timeout)
+
         if self.server.ssl:
             try:
                 cert, key, _ = self.server.ssloptions.get_cert(None)
                 self.convert_to_ssl(
                     cert,
                     key,
-                    handle_sni=self.handle_sni,
+                    dhparams=self.server.ssloptions.certstore.dhparams,
+                    handle_sni=self._handle_sni,
                     request_client_cert=self.server.ssloptions.request_client_cert,
                     cipher_list=self.server.ssloptions.ciphers,
                     method=self.server.ssloptions.sslversion,
@@ -356,8 +411,20 @@ class PathodHandler(tcp.BaseHandler):
                 )
                 log.write(self.logfp, s)
                 return
-        self.settimeout(self.server.timeout)
+
+            alp = self.get_alpn_proto_negotiated()
+            if alp == http2.HTTP2Protocol.ALPN_PROTO_H2:
+                self.protocol = http2.HTTP2Protocol(self, is_server=True, dump_frames=self.http2_framedump)
+                self.use_http2 = True
+
+        # if not self.protocol:
+        #     # TODO: create HTTP or Websockets protocol
+        #     self.protocol = None
+
+        self.settings.protocol = self.protocol
+
         handler = self.handle_http_request
+
         while not self.finished:
             handler, l = handler()
             if l:
@@ -365,6 +432,18 @@ class PathodHandler(tcp.BaseHandler):
             if not handler:
                 return
 
+    def addlog(self, log):
+        # FIXME: The bytes in the log should not be escaped. We do this at the
+        # moment because JSON encoding can't handle binary data, and I don't
+        # want to base64 everything.
+        if self.server.logreq:
+            bytes = self.rfile.get_log().encode("string_escape")
+            log["request_bytes"] = bytes
+        if self.server.logresp:
+            bytes = self.wfile.get_log().encode("string_escape")
+            log["response_bytes"] = bytes
+        self.server.add_log(log)
+
 
 class Pathod(tcp.TCPServer):
     LOGBUF = 500
@@ -387,6 +466,7 @@ class Pathod(tcp.TCPServer):
         logresp=False,
         explain=False,
         hexdump=False,
+        http2_framedump=False,
         webdebug=False,
         logfp=sys.stdout,
     ):
@@ -414,6 +494,7 @@ class Pathod(tcp.TCPServer):
         self.noapi, self.nohang = noapi, nohang
         self.timeout, self.logreq = timeout, logreq
         self.logresp, self.hexdump = logresp, hexdump
+        self.http2_framedump = http2_framedump
         self.explain = explain
         self.logfp = logfp
 
@@ -451,7 +532,8 @@ class Pathod(tcp.TCPServer):
             client_address,
             self,
             self.logfp,
-            self.settings
+            self.settings,
+            self.http2_framedump,
         )
         try:
             h.handle()
@@ -508,7 +590,6 @@ def main(args):  # pragma: nocover
         sslversion = utils.SSLVERSIONS[args.sslversion],
         certs = args.ssl_certs,
         sans = args.sans,
-        alpn_select = args.alpn_select,
     )
 
     root = logging.getLogger()
@@ -548,6 +629,7 @@ def main(args):  # pragma: nocover
             logreq = args.logreq,
             logresp = args.logresp,
             hexdump = args.hexdump,
+            http2_framedump = args.http2_framedump,
             explain = args.explain,
             webdebug = args.webdebug
         )

libpathod/pathod_cmdline.py

@@ -139,10 +139,11 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr):
         """
     )
     group.add_argument(
-        "--sslversion", dest="sslversion", type=int, default=4,
+        "--sslversion", dest="sslversion", type=str, default='SSLv23',
-        choices=[1, 2, 3, 4],
+        choices=utils.SSLVERSIONS.keys(),
-        help=""""Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
+        help=""""
-        to SSLv23."""
+            Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23.
+            Default to SSLv23."""
     )
 
     group = parser.add_argument_group(
@@ -172,6 +173,12 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr):
         "-x", dest="hexdump", action="store_true", default=False,
         help="Log request/response in hexdump format"
     )
+    group.add_argument(
+        "--http2-framedump", dest="http2_framedump", action="store_true", default=False,
+        help="Output all received & sent HTTP/2 frames"
+    )
+
+
     args = parser.parse_args(argv[1:])
 
     certs = []

libpathod/utils.py

@@ -3,10 +3,12 @@ import sys
 from netlib import tcp
 
 SSLVERSIONS = {
-    1: tcp.TLSv1_METHOD,
+    'TLSv1.2': tcp.TLSv1_2_METHOD,
-    2: tcp.SSLv2_METHOD,
+    'TLSv1.1': tcp.TLSv1_1_METHOD,
-    3: tcp.SSLv3_METHOD,
+    'TLSv1':   tcp.TLSv1_METHOD,
-    4: tcp.SSLv23_METHOD,
+    'SSLv3':   tcp.SSLv3_METHOD,
+    'SSLv2':   tcp.SSLv2_METHOD,
+    'SSLv23':  tcp.SSLv23_METHOD,
 }
 
 SIZE_UNITS = dict(

test/test_language_http2.py

@@ -1,5 +1,6 @@
 import cStringIO
 
+from netlib import tcp
 from libpathod import language
 from libpathod.language import http2, base
 import netlib
@@ -64,7 +65,7 @@ class TestRequest:
             s,
             language.Settings(
                 request_host = "foo.com",
-                protocol = netlib.http2.HTTP2Protocol(None)
+                protocol = netlib.http2.HTTP2Protocol(tcp.TCPClient(('localhost', 1234)))
             )
         )
 

test/test_pathoc.py

@@ -230,9 +230,6 @@ class TestDaemon(_TestDaemon):
 
 class TestDaemonHTTP2(_TestDaemon):
     ssl = True
-    ssloptions = pathod.SSLOptions(
-        alpn_select = http2.HTTP2Protocol.ALPN_PROTO_H2,
-    )
 
     def test_http2(self):
         c = pathoc.Pathoc(
@@ -270,5 +267,5 @@ class TestDaemonHTTP2(_TestDaemon):
             use_http2 = True,
         )
         c.connect()
-        resp = c.request("get:/api/info")
+        resp = c.request("get:/p/200")
-        assert tuple(json.loads(resp.content)["version"]) == version.IVERSION
+        assert resp.status_code == "200"