From 6be65954259c845f594ea4f4515b928d0c1ee24c Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 8 Jun 2015 17:12:51 +0200 Subject: [PATCH 01/10] pathod: add ALPN support --- libpathod/pathod.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 13f602b4d..505c8c22c 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -39,21 +39,23 @@ class SSLOptions: request_client_cert=False, sslversion=tcp.SSLv23_METHOD, ciphers=None, - certs=None + certs=None, + alpn_select=None, ): self.confdir = confdir self.cn = cn + self.sans = sans + self.not_after_connect = not_after_connect + self.request_client_cert = request_client_cert + self.sslversion = sslversion + self.ciphers = ciphers + self.alpn_select = alpn_select self.certstore = certutils.CertStore.from_store( os.path.expanduser(confdir), CERTSTORE_BASENAME ) for i in certs or []: self.certstore.add_cert_file(*i) - self.not_after_connect = not_after_connect - self.request_client_cert = request_client_cert - self.ciphers = ciphers - self.sslversion = sslversion - self.sans = sans def get_cert(self, name): if self.cn: @@ -173,6 +175,7 @@ class PathodHandler(tcp.BaseHandler): request_client_cert=self.server.ssloptions.request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.sslversion, + alpn_select=self.server.ssloptions.alpn_select, ) except tcp.NetLibError as v: s = str(v) @@ -340,6 +343,7 @@ class PathodHandler(tcp.BaseHandler): request_client_cert=self.server.ssloptions.request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.sslversion, + alpn_select=self.server.ssloptions.alpn_select, ) except tcp.NetLibError as v: s = str(v) @@ -502,7 +506,8 @@ def main(args): # pragma: nocover ciphers = args.ciphers, sslversion = utils.SSLVERSIONS[args.sslversion], certs = args.ssl_certs, - sans = args.sans + sans = args.sans, + alpn_select = args.alpn_select, ) root = logging.getLogger() From 9ececa637d69ee3991fbc82f3cd64ae781a86f5d Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 8 Jun 2015 15:28:19 +0200 Subject: [PATCH 02/10] http2: add warning for missing ALPN support --- libpathod/pathoc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index ba06b2f1a..e9bd5f569 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -216,6 +216,10 @@ class Pathoc(tcp.TCPClient): self.ws_framereader = None if self.use_http2: + if not OpenSSL._util.lib.Cryptography_HAS_ALPN: # pragma: nocover + print >> sys.stderr, "HTTP/2 requires ALPN support. Please use OpenSSL >= 1.0.2." + print >> sys.stderr, "Pathoc might not be working as expected without ALPN." + self.protocol = http2.HTTP2Protocol(self) else: # TODO: create HTTP or Websockets protocol @@ -259,7 +263,7 @@ class Pathoc(tcp.TCPClient): an HTTP CONNECT request. """ if self.use_http2 and not self.ssl: - raise ValueError("HTTP2 without SSL is not supported.") + raise NotImplementedError("HTTP2 without SSL is not supported.") tcp.TCPClient.connect(self) From 18a4456397d0b4b1275ac2c8ab393d041176e949 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 8 Jun 2015 16:03:33 +0200 Subject: [PATCH 03/10] refactor cmdline tests --- libpathod/pathoc_cmdline.py | 218 ++++++++++++++++++ libpathod/{cmdline.py => pathod_cmdline.py} | 207 ----------------- pathoc | 3 +- pathod | 3 +- setup.py | 4 +- test/test_pathoc_cmdline.py | 62 +++++ ...test_cmdline.py => test_pathod_cmdline.py} | 53 +---- 7 files changed, 287 insertions(+), 263 deletions(-) create mode 100644 libpathod/pathoc_cmdline.py rename libpathod/{cmdline.py => pathod_cmdline.py} (52%) create mode 100644 test/test_pathoc_cmdline.py rename test/{test_cmdline.py => test_pathod_cmdline.py} (54%) diff --git a/libpathod/pathoc_cmdline.py b/libpathod/pathoc_cmdline.py new file mode 100644 index 000000000..fa30aa1e2 --- /dev/null +++ b/libpathod/pathoc_cmdline.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python +import sys +import argparse +import os +import os.path +import re +from netlib import http_uastrings +from . import pathoc, pathod, version, utils, language + + +def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): + preparser = argparse.ArgumentParser(add_help=False) + preparser.add_argument( + "--show-uas", dest="showua", action="store_true", default=False, + help="Print user agent shortcuts and exit." + ) + pa = preparser.parse_known_args(argv)[0] + if pa.showua: + print >> stdout, "User agent strings:" + for i in http_uastrings.UASTRINGS: + print >> stdout, " ", i[1], i[0] + sys.exit(0) + + parser = argparse.ArgumentParser( + description='A perverse HTTP client.', parents=[preparser] + ) + parser.add_argument( + '--version', + action='version', + version="pathoc " + version.VERSION + ) + parser.add_argument( + "-c", dest="connect_to", type=str, default=False, + metavar = "HOST:PORT", + help="Issue an HTTP CONNECT to connect to the specified host." + ) + parser.add_argument( + "--memo-limit", dest='memolimit', default=5000, type=int, metavar="N", + help='Stop if we do not find a valid request after N attempts.' + ) + parser.add_argument( + "-m", dest='memo', action="store_true", default=False, + help=""" + Remember specs, and never play the same one twice. Note that this + means requests have to be rendered in memory, which means that + large generated data can cause issues. + """ + ) + parser.add_argument( + "-n", dest='repeat', default=1, type=int, metavar="N", + help='Repeat N times. If 0 repeat for ever.' + ) + parser.add_argument( + "-w", dest='wait', default=0, type=float, metavar="N", + help='Wait N seconds between each request.' + ) + parser.add_argument( + "-r", dest="random", action="store_true", default=False, + help=""" + Select a random request from those specified. If this is not specified, + requests are all played in sequence. + """ + ) + parser.add_argument( + "-t", dest="timeout", type=int, default=None, + help="Connection timeout" + ) + parser.add_argument( + "--http2", dest="use_http2", action="store_true", default=False, + help='Perform all requests over a single HTTP/2 connection.' + ) + parser.add_argument( + "--http2-skip-connection-preface", + dest="http2_skip_connection_preface", + action="store_true", + default=False, + help='Skips the HTTP/2 connection preface before sending requests.') + + parser.add_argument( + 'host', type=str, + metavar = "host[:port]", + help='Host and port to connect to' + ) + parser.add_argument( + 'requests', type=str, nargs="+", + help=""" + Request specification, or path to a file containing request + specifcations + """ + ) + + group = parser.add_argument_group( + 'SSL', + ) + group.add_argument( + "-s", dest="ssl", action="store_true", default=False, + help="Connect with SSL" + ) + group.add_argument( + "-C", dest="clientcert", type=str, default=False, + help="Path to a file containing client certificate and private key" + ) + group.add_argument( + "-i", dest="sni", type=str, default=False, + help="SSL Server Name Indication" + ) + group.add_argument( + "--ciphers", dest="ciphers", type=str, default=False, + help="SSL cipher specification" + ) + group.add_argument( + "--sslversion", dest="sslversion", type=int, default=4, + choices=[1, 2, 3, 4], + help=""" + Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default + to SSLv23. + """ + ) + + group = parser.add_argument_group( + 'Controlling Output', + """ + Some of these options expand generated values for logging - if + you're generating large data, use them with caution. + """ + ) + group.add_argument( + "-I", dest="ignorecodes", type=str, default="", + help="Comma-separated list of response codes to ignore" + ) + group.add_argument( + "-S", dest="showssl", action="store_true", default=False, + help="Show info on SSL connection" + ) + group.add_argument( + "-e", dest="explain", action="store_true", default=False, + help="Explain requests" + ) + group.add_argument( + "-o", dest="oneshot", action="store_true", default=False, + help="Oneshot - exit after first non-ignored response" + ) + group.add_argument( + "-q", dest="showreq", action="store_true", default=False, + help="Print full request" + ) + group.add_argument( + "-p", dest="showresp", action="store_true", default=False, + help="Print full response" + ) + group.add_argument( + "-T", dest="ignoretimeout", action="store_true", default=False, + help="Ignore timeouts" + ) + group.add_argument( + "-x", dest="hexdump", action="store_true", default=False, + help="Output in hexdump format" + ) + + args = parser.parse_args(argv[1:]) + + args.port = None + if ":" in args.host: + h, p = args.host.rsplit(":", 1) + try: + p = int(p) + except ValueError: + return parser.error("Invalid port in host spec: %s" % args.host) + args.host = h + args.port = p + + if args.port is None: + args.port = 443 if args.ssl else 80 + + try: + args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i] + except ValueError: + return parser.error( + "Invalid return code specification: %s" % + args.ignorecodes) + + if args.connect_to: + parts = args.connect_to.split(":") + if len(parts) != 2: + return parser.error( + "Invalid CONNECT specification: %s" % + args.connect_to) + try: + parts[1] = int(parts[1]) + except ValueError: + return parser.error( + "Invalid CONNECT specification: %s" % + args.connect_to) + args.connect_to = parts + else: + args.connect_to = None + + if args.http2_skip_connection_preface: + args.use_http2 = True + + reqs = [] + for r in args.requests: + if os.path.isfile(r): + data = open(r).read() + r = data + try: + reqs.append(language.parse_pathoc(r, args.use_http2)) + except language.ParseException as v: + print >> stderr, "Error parsing request spec: %s" % v.msg + print >> stderr, v.marked() + sys.exit(1) + args.requests = reqs + return args + + +def go_pathoc(): # pragma: nocover + args = args_pathoc(sys.argv) + pathoc.main(args) diff --git a/libpathod/cmdline.py b/libpathod/pathod_cmdline.py similarity index 52% rename from libpathod/cmdline.py rename to libpathod/pathod_cmdline.py index 06a6c533c..c1f016c23 100644 --- a/libpathod/cmdline.py +++ b/libpathod/pathod_cmdline.py @@ -8,213 +8,6 @@ from netlib import http_uastrings from . import pathoc, pathod, version, utils, language -def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): - preparser = argparse.ArgumentParser(add_help=False) - preparser.add_argument( - "--show-uas", dest="showua", action="store_true", default=False, - help="Print user agent shortcuts and exit." - ) - pa = preparser.parse_known_args(argv)[0] - if pa.showua: - print >> stdout, "User agent strings:" - for i in http_uastrings.UASTRINGS: - print >> stdout, " ", i[1], i[0] - sys.exit(0) - - parser = argparse.ArgumentParser( - description='A perverse HTTP client.', parents=[preparser] - ) - parser.add_argument( - '--version', - action='version', - version="pathoc " + version.VERSION - ) - parser.add_argument( - "-c", dest="connect_to", type=str, default=False, - metavar = "HOST:PORT", - help="Issue an HTTP CONNECT to connect to the specified host." - ) - parser.add_argument( - "--memo-limit", dest='memolimit', default=5000, type=int, metavar="N", - help='Stop if we do not find a valid request after N attempts.' - ) - parser.add_argument( - "-m", dest='memo', action="store_true", default=False, - help=""" - Remember specs, and never play the same one twice. Note that this - means requests have to be rendered in memory, which means that - large generated data can cause issues. - """ - ) - parser.add_argument( - "-n", dest='repeat', default=1, type=int, metavar="N", - help='Repeat N times. If 0 repeat for ever.' - ) - parser.add_argument( - "-w", dest='wait', default=0, type=float, metavar="N", - help='Wait N seconds between each request.' - ) - parser.add_argument( - "-r", dest="random", action="store_true", default=False, - help=""" - Select a random request from those specified. If this is not specified, - requests are all played in sequence. - """ - ) - parser.add_argument( - "-t", dest="timeout", type=int, default=None, - help="Connection timeout" - ) - parser.add_argument( - "--http2", dest="use_http2", action="store_true", default=False, - help='Perform all requests over a single HTTP/2 connection.' - ) - parser.add_argument( - "--http2-skip-connection-preface", - dest="http2_skip_connection_preface", - action="store_true", - default=False, - help='Skips the HTTP/2 connection preface before sending requests.') - - parser.add_argument( - 'host', type=str, - metavar = "host[:port]", - help='Host and port to connect to' - ) - parser.add_argument( - 'requests', type=str, nargs="+", - help=""" - Request specification, or path to a file containing request - specifcations - """ - ) - - group = parser.add_argument_group( - 'SSL', - ) - group.add_argument( - "-s", dest="ssl", action="store_true", default=False, - help="Connect with SSL" - ) - group.add_argument( - "-C", dest="clientcert", type=str, default=False, - help="Path to a file containing client certificate and private key" - ) - group.add_argument( - "-i", dest="sni", type=str, default=False, - help="SSL Server Name Indication" - ) - group.add_argument( - "--ciphers", dest="ciphers", type=str, default=False, - help="SSL cipher specification" - ) - group.add_argument( - "--sslversion", dest="sslversion", type=int, default=4, - choices=[1, 2, 3, 4], - help=""" - Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default - to SSLv23. - """ - ) - - group = parser.add_argument_group( - 'Controlling Output', - """ - Some of these options expand generated values for logging - if - you're generating large data, use them with caution. - """ - ) - group.add_argument( - "-I", dest="ignorecodes", type=str, default="", - help="Comma-separated list of response codes to ignore" - ) - group.add_argument( - "-S", dest="showssl", action="store_true", default=False, - help="Show info on SSL connection" - ) - group.add_argument( - "-e", dest="explain", action="store_true", default=False, - help="Explain requests" - ) - group.add_argument( - "-o", dest="oneshot", action="store_true", default=False, - help="Oneshot - exit after first non-ignored response" - ) - group.add_argument( - "-q", dest="showreq", action="store_true", default=False, - help="Print full request" - ) - group.add_argument( - "-p", dest="showresp", action="store_true", default=False, - help="Print full response" - ) - group.add_argument( - "-T", dest="ignoretimeout", action="store_true", default=False, - help="Ignore timeouts" - ) - group.add_argument( - "-x", dest="hexdump", action="store_true", default=False, - help="Output in hexdump format" - ) - - args = parser.parse_args(argv[1:]) - - args.port = None - if ":" in args.host: - h, p = args.host.rsplit(":", 1) - try: - p = int(p) - except ValueError: - return parser.error("Invalid port in host spec: %s" % args.host) - args.host = h - args.port = p - - if args.port is None: - args.port = 443 if args.ssl else 80 - - try: - args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i] - except ValueError: - return parser.error( - "Invalid return code specification: %s" % - args.ignorecodes) - - if args.connect_to: - parts = args.connect_to.split(":") - if len(parts) != 2: - return parser.error( - "Invalid CONNECT specification: %s" % - args.connect_to) - try: - parts[1] = int(parts[1]) - except ValueError: - return parser.error( - "Invalid CONNECT specification: %s" % - args.connect_to) - args.connect_to = parts - else: - args.connect_to = None - - reqs = [] - for r in args.requests: - if os.path.isfile(r): - data = open(r).read() - r = data - try: - reqs.append(language.parse_pathoc(r, args.use_http2)) - except language.ParseException as v: - print >> stderr, "Error parsing request spec: %s" % v.msg - print >> stderr, v.marked() - sys.exit(1) - args.requests = reqs - return args - - -def go_pathoc(): # pragma: nocover - args = args_pathoc(sys.argv) - pathoc.main(args) - - def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr): parser = argparse.ArgumentParser( description='A pathological HTTP/S daemon.' diff --git a/pathoc b/pathoc index cbf8f7738..b31216113 100755 --- a/pathoc +++ b/pathoc @@ -1,5 +1,6 @@ #!/usr/bin/env python -from libpathod import cmdline + +from libpathod import pathoc_cmdline as cmdline if __name__ == "__main__": cmdline.go_pathoc() diff --git a/pathod b/pathod index ca0baa573..a79becf1f 100755 --- a/pathod +++ b/pathod @@ -1,5 +1,6 @@ #!/usr/bin/env python -from libpathod import cmdline + +from libpathod import pathod_cmdline as cmdline if __name__ == "__main__": cmdline.go_pathod() diff --git a/setup.py b/setup.py index 7340f983a..1a6a3e3f1 100644 --- a/setup.py +++ b/setup.py @@ -37,8 +37,8 @@ setup( include_package_data=True, entry_points={ 'console_scripts': [ - "pathod = libpathod.cmdline:go_pathod", - "pathoc = libpathod.cmdline:go_pathoc" + "pathod = libpathod.pathod_cmdline:go_pathod", + "pathoc = libpathod.pathoc_cmdline:go_pathoc" ] }, install_requires=[ diff --git a/test/test_pathoc_cmdline.py b/test/test_pathoc_cmdline.py new file mode 100644 index 000000000..03c838446 --- /dev/null +++ b/test/test_pathoc_cmdline.py @@ -0,0 +1,62 @@ +from libpathod import pathoc_cmdline as cmdline +import tutils +import cStringIO +import mock + + +@mock.patch("argparse.ArgumentParser.error") +def test_pathoc(perror): + assert cmdline.args_pathoc(["pathoc", "foo.com", "get:/"]) + s = cStringIO.StringIO() + tutils.raises( + SystemExit, cmdline.args_pathoc, [ + "pathoc", "--show-uas"], s, s) + + a = cmdline.args_pathoc(["pathoc", "foo.com:8888", "get:/"]) + assert a.port == 8888 + + a = cmdline.args_pathoc(["pathoc", "foo.com:xxx", "get:/"]) + assert perror.called + perror.reset_mock() + + a = cmdline.args_pathoc(["pathoc", "-I", "10, 20", "foo.com:8888", "get:/"]) + assert a.ignorecodes == [10, 20] + + a = cmdline.args_pathoc(["pathoc", "-I", "xx, 20", "foo.com:8888", "get:/"]) + assert perror.called + perror.reset_mock() + + a = cmdline.args_pathoc(["pathoc", "-c", "foo:10", "foo.com:8888", "get:/"]) + assert a.connect_to == ["foo", 10] + + a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2"]) + assert a.use_http2 == True + + a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2-skip-connection-preface"]) + assert a.use_http2 == True + assert a.http2_skip_connection_preface == True + + a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"]) + assert perror.called + perror.reset_mock() + + a = cmdline.args_pathoc( + ["pathoc", "-c", "foo:bar", "foo.com:8888", "get:/"]) + assert perror.called + perror.reset_mock() + + a = cmdline.args_pathoc( + [ + "pathoc", + "foo.com:8888", + tutils.test_data.path("data/request") + ] + ) + assert len(list(a.requests)) == 1 + + tutils.raises( + SystemExit, + cmdline.args_pathoc, + ["pathoc", "foo.com", "invalid"], + s, s + ) diff --git a/test/test_cmdline.py b/test/test_pathod_cmdline.py similarity index 54% rename from test/test_cmdline.py rename to test/test_pathod_cmdline.py index c1b556082..590bb56bc 100644 --- a/test/test_cmdline.py +++ b/test/test_pathod_cmdline.py @@ -1,4 +1,4 @@ -from libpathod import cmdline +from libpathod import pathod_cmdline as cmdline import tutils import cStringIO import mock @@ -93,50 +93,6 @@ def test_pathod(perror): assert perror.called perror.reset_mock() - -@mock.patch("argparse.ArgumentParser.error") -def test_pathoc(perror): - assert cmdline.args_pathoc(["pathoc", "foo.com", "get:/"]) - s = cStringIO.StringIO() - tutils.raises( - SystemExit, cmdline.args_pathoc, [ - "pathoc", "--show-uas"], s, s) - - a = cmdline.args_pathoc(["pathoc", "foo.com:8888", "get:/"]) - assert a.port == 8888 - - a = cmdline.args_pathoc(["pathoc", "foo.com:xxx", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc(["pathoc", "-I", "10, 20", "foo.com:8888", "get:/"]) - assert a.ignorecodes == [10, 20] - - a = cmdline.args_pathoc(["pathoc", "-I", "xx, 20", "foo.com:8888", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc(["pathoc", "-c", "foo:10", "foo.com:8888", "get:/"]) - assert a.connect_to == ["foo", 10] - - a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc( - ["pathoc", "-c", "foo:bar", "foo.com:8888", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc( - [ - "pathoc", - "foo.com:8888", - tutils.test_data.path("data/request") - ] - ) - assert len(list(a.requests)) == 1 - a = cmdline.args_pathod( [ "pathod", @@ -146,10 +102,3 @@ def test_pathoc(perror): ) assert perror.called perror.reset_mock() - - tutils.raises( - SystemExit, - cmdline.args_pathoc, - ["pathoc", "foo.com", "invalid"], - s, s - ) From b7c8021407965cd97bcc8051ddb0692145401cbf Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 8 Jun 2015 17:12:01 +0200 Subject: [PATCH 04/10] http2: using http2 implies SSL --- libpathod/pathoc_cmdline.py | 3 +++ test/test_pathoc_cmdline.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/libpathod/pathoc_cmdline.py b/libpathod/pathoc_cmdline.py index fa30aa1e2..dcd75d117 100644 --- a/libpathod/pathoc_cmdline.py +++ b/libpathod/pathoc_cmdline.py @@ -198,6 +198,9 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): if args.http2_skip_connection_preface: args.use_http2 = True + if args.use_http2: + args.ssl = True + reqs = [] for r in args.requests: if os.path.isfile(r): diff --git a/test/test_pathoc_cmdline.py b/test/test_pathoc_cmdline.py index 03c838446..6c070aed9 100644 --- a/test/test_pathoc_cmdline.py +++ b/test/test_pathoc_cmdline.py @@ -31,9 +31,11 @@ def test_pathoc(perror): a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2"]) assert a.use_http2 == True + assert a.ssl == True a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2-skip-connection-preface"]) assert a.use_http2 == True + assert a.ssl == True assert a.http2_skip_connection_preface == True a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"]) From 0bc8fa1d0d4f2730311b9fc934a8fcdaae07ab66 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 8 Jun 2015 15:28:24 +0200 Subject: [PATCH 05/10] http2: add pathoc and language tests --- libpathod/language/http2.py | 8 +--- test/test_language_http2.py | 75 +++++++++++++++++++++++++++++++++++++ test/test_pathoc.py | 62 ++++++++++++++++++++++++++++-- 3 files changed, 136 insertions(+), 9 deletions(-) create mode 100644 test/test_language_http2.py diff --git a/libpathod/language/http2.py b/libpathod/language/http2.py index d78fc5c86..4a5b90840 100644 --- a/libpathod/language/http2.py +++ b/libpathod/language/http2.py @@ -102,7 +102,7 @@ class Request(message.Message): def values(self, settings): return settings.protocol.create_request( - self.method.value.get_generator(settings), + self.method.string(), self.path, self.headers, self.body) @@ -111,9 +111,5 @@ class Request(message.Message): return ":".join([i.spec() for i in self.tokens]) -# class H2F(base.CaselessLiteral): -# TOK = "h2f" -# -# -# class WebsocketFrame(message.Message): +# class Frame(message.Message): # pass diff --git a/test/test_language_http2.py b/test/test_language_http2.py new file mode 100644 index 000000000..de3e5cf93 --- /dev/null +++ b/test/test_language_http2.py @@ -0,0 +1,75 @@ +import cStringIO + +from libpathod import language +from libpathod.language import http2, base +import netlib +import tutils + + +def parse_request(s): + return language.parse_pathoc(s, True).next() + + +class TestRequest: + def test_nonascii(self): + tutils.raises("ascii", parse_request, "get:\xf0") + + def test_err(self): + tutils.raises(language.ParseException, parse_request, 'GET') + + def test_simple(self): + r = parse_request('GET:"/foo"') + assert r.method.string() == "GET" + assert r.path.string() == "/foo" + r = parse_request('GET:/foo') + assert r.path.string() == "/foo" + + def test_multiple(self): + r = list(language.parse_pathoc("GET:/ PUT:/")) + assert r[0].method.string() == "GET" + assert r[1].method.string() == "PUT" + assert len(r) == 2 + + l = """ + GET + "/foo" + + PUT + + "/foo + + + + bar" + """ + r = list(language.parse_pathoc(l, True)) + assert len(r) == 2 + assert r[0].method.string() == "GET" + assert r[1].method.string() == "PUT" + + l = """ + get:"http://localhost:9999/p/200" + get:"http://localhost:9999/p/200" + """ + r = list(language.parse_pathoc(l, True)) + assert len(r) == 2 + assert r[0].method.string() == "GET" + assert r[1].method.string() == "GET" + + def test_render(self): + s = cStringIO.StringIO() + r = parse_request("GET:'/foo'") + assert language.serve( + r, + s, + language.Settings( + request_host = "foo.com", + protocol = netlib.http2.HTTP2Protocol(None) + ) + ) + + def test_spec(self): + def rt(s): + s = parse_request(s).spec() + assert parse_request(s).spec() == s + rt("get:/foo") diff --git a/test/test_pathoc.py b/test/test_pathoc.py index c00c7d535..28514378b 100644 --- a/test/test_pathoc.py +++ b/test/test_pathoc.py @@ -1,8 +1,9 @@ import json import cStringIO import re +from mock import Mock -from netlib import tcp, http +from netlib import tcp, http, http2 from libpathod import pathoc, test, version, pathod, language import tutils @@ -86,8 +87,9 @@ class _TestDaemon: class TestDaemonSSL(_TestDaemon): ssl = True ssloptions = pathod.SSLOptions( - request_client_cert=True, - sans = ["test1.com", "test2.com"] + request_client_cert = True, + sans = ["test1.com", "test2.com"], + alpn_select = http2.HTTP2Protocol.ALPN_PROTO_H2, ) def test_sni(self): @@ -119,6 +121,14 @@ class TestDaemonSSL(_TestDaemon): d = json.loads(r.content) assert d["log"][0]["request"]["clientcert"]["keyinfo"] + def test_http2_without_ssl(self): + c = pathoc.Pathoc( + ("127.0.0.1", self.d.port), + use_http2 = True, + ssl = False, + ) + tutils.raises(NotImplementedError, c.connect) + class TestDaemon(_TestDaemon): ssl = False @@ -216,3 +226,49 @@ class TestDaemon(_TestDaemon): "HTTP/1.1 200 OK\r\n" ) c.http_connect(to) + + +class TestDaemonHTTP2(_TestDaemon): + ssl = True + ssloptions = pathod.SSLOptions( + alpn_select = http2.HTTP2Protocol.ALPN_PROTO_H2, + ) + + def test_http2(self): + c = pathoc.Pathoc( + ("127.0.0.1", self.d.port), + use_http2 = True, + ssl = True, + ) + assert isinstance(c.protocol, http2.HTTP2Protocol) + + c = pathoc.Pathoc( + ("127.0.0.1", self.d.port), + ) + assert c.protocol == None # TODO: change if other protocols get implemented + + def test_http2_alpn(self): + c = pathoc.Pathoc( + ("127.0.0.1", self.d.port), + ssl = True, + use_http2 = True, + http2_skip_connection_preface = True, + ) + + tmp_convert_to_ssl = c.convert_to_ssl + c.convert_to_ssl = Mock() + c.convert_to_ssl.side_effect = tmp_convert_to_ssl + c.connect() + + _, kwargs = c.convert_to_ssl.call_args + assert set(kwargs['alpn_protos']) == set([b'http1.1', b'h2']) + + def test_request(self): + c = pathoc.Pathoc( + ("127.0.0.1", self.d.port), + ssl = True, + use_http2 = True, + ) + c.connect() + resp = c.request("get:/api/info") + assert tuple(json.loads(resp.content)["version"]) == version.IVERSION From 22811c45dd1e6e6f1c8108e83a7be625f305c19e Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 11 Jun 2015 16:36:58 +0200 Subject: [PATCH 06/10] fix craft anchor The go-button in the app was broken due to an invalid string representation of the regex. A plain string used as URL prefix simplifies this drastically. --- libpathod/pathod.py | 11 ++++++----- libpathod/pathod_cmdline.py | 13 +++---------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 505c8c22c..f14961810 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -20,7 +20,7 @@ DEFAULT_CERT_DOMAIN = "pathod.net" CONFDIR = "~/.mitmproxy" CERTSTORE_BASENAME = "mitmproxy" CA_CERT_NAME = "mitmproxy-ca.pem" -DEFAULT_ANCHOR = r"/p/?" +DEFAULT_CRAFT_ANCHOR = "/p/" logger = logging.getLogger('pathod') @@ -293,8 +293,9 @@ class PathodHandler(tcp.BaseHandler): anchor_gen = i[1] break else: - if m(self.server.craftanchor.match(path)): - spec = urllib.unquote(path)[len(m.v.group()):] + print(self.server.craftanchor) + if m(path.startswith(self.server.craftanchor)): + spec = urllib.unquote(path)[len(self.server.craftanchor):] if spec: try: anchor_gen = language.parse_pathod(spec) @@ -373,7 +374,7 @@ class Pathod(tcp.TCPServer): addr, ssl=False, ssloptions=None, - craftanchor=re.compile(DEFAULT_ANCHOR), + craftanchor=DEFAULT_CRAFT_ANCHOR, staticdir=None, anchors=(), sizelimit=None, @@ -393,7 +394,7 @@ class Pathod(tcp.TCPServer): addr: (address, port) tuple. If port is 0, a free port will be automatically chosen. ssloptions: an SSLOptions object. - craftanchor: string specifying the path under which to anchor + craftanchor: URL prefix specifying the path under which to anchor response generation. staticdir: path to a directory of static resources, or None. anchors: List of (regex object, language.Request object) tuples, or diff --git a/libpathod/pathod_cmdline.py b/libpathod/pathod_cmdline.py index c1f016c23..68828aca9 100644 --- a/libpathod/pathod_cmdline.py +++ b/libpathod/pathod_cmdline.py @@ -45,11 +45,11 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr): """ ) parser.add_argument( - "-c", dest='craftanchor', default=pathod.DEFAULT_ANCHOR, type=str, + "-c", dest='craftanchor', default=pathod.DEFAULT_CRAFT_ANCHOR, type=str, help=""" - Regular expression specifying anchor point for URL crafting + URL path specifying prefix for URL crafting commands. (%s) - """%pathod.DEFAULT_ANCHOR + """%pathod.DEFAULT_CRAFT_ANCHOR ) parser.add_argument( "--confdir", @@ -203,13 +203,6 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr): return parser.error(v) args.sizelimit = sizelimit - try: - args.craftanchor = re.compile(args.craftanchor) - except re.error: - return parser.error( - "Invalid regex in craft anchor: %s" % args.craftanchor - ) - anchors = [] for patt, spec in args.anchors: if os.path.isfile(spec): From 30fbf57e4b72e3947c323d98aee7b2d44663e33c Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 12 Jun 2015 13:41:04 +0200 Subject: [PATCH 07/10] delay pathod parsing until needed This allows us to use different languages based on runtime env. --- libpathod/pathod_cmdline.py | 8 +------- test/test_pathoc.py | 2 +- test/test_pathod_cmdline.py | 19 ------------------- test/tutils.py | 2 +- 4 files changed, 3 insertions(+), 28 deletions(-) diff --git a/libpathod/pathod_cmdline.py b/libpathod/pathod_cmdline.py index 68828aca9..f1bb6982a 100644 --- a/libpathod/pathod_cmdline.py +++ b/libpathod/pathod_cmdline.py @@ -208,17 +208,11 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr): if os.path.isfile(spec): data = open(spec).read() spec = data - try: - req = language.parse_pathod(spec) - except language.ParseException as v: - print >> stderr, "Error parsing anchor spec: %s" % v.msg - print >> stderr, v.marked() - sys.exit(1) try: arex = re.compile(patt) except re.error: return parser.error("Invalid regex in anchor: %s" % patt) - anchors.append((arex, req)) + anchors.append((arex, spec)) args.anchors = anchors return args diff --git a/test/test_pathoc.py b/test/test_pathoc.py index 28514378b..e1e1fe97b 100644 --- a/test/test_pathoc.py +++ b/test/test_pathoc.py @@ -23,7 +23,7 @@ class _TestDaemon: ssloptions = self.ssloptions, staticdir = tutils.test_data.path("data"), anchors = [ - (re.compile("/anchor/.*"), language.parse_pathod("202")) + (re.compile("/anchor/.*"), "202") ] ) diff --git a/test/test_pathod_cmdline.py b/test/test_pathod_cmdline.py index 590bb56bc..829c4b32e 100644 --- a/test/test_pathod_cmdline.py +++ b/test/test_pathod_cmdline.py @@ -65,15 +65,6 @@ def test_pathod(perror): assert perror.called perror.reset_mock() - s = cStringIO.StringIO() - tutils.raises( - SystemExit, - cmdline.args_pathod, - ["pathod", "-a", "foo=."], - s, - s - ) - a = cmdline.args_pathod( [ "pathod", @@ -92,13 +83,3 @@ def test_pathod(perror): ) assert perror.called perror.reset_mock() - - a = cmdline.args_pathod( - [ - "pathod", - "-c", - "?" - ] - ) - assert perror.called - perror.reset_mock() diff --git a/test/tutils.py b/test/tutils.py index c56c60d43..60c0765a5 100644 --- a/test/tutils.py +++ b/test/tutils.py @@ -27,7 +27,7 @@ class DaemonTests(object): klass.d = test.Daemon( staticdir=test_data.path("data"), anchors=[ - (re.compile("/anchor/.*"), language.parse_pathod("202:da")) + (re.compile("/anchor/.*"), "202:da") ], ssl = klass.ssl, ssloptions = so, From a0d8afd0fcc3c678da0dc956c5a80d4e07d5ac3e Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 11 Jun 2015 16:13:22 +0200 Subject: [PATCH 08/10] http2: add request-response to pathod --- libpathod/language/__init__.py | 14 ++- libpathod/language/http.py | 6 +- libpathod/language/http2.py | 115 ++++++++++++++++-- libpathod/pathoc.py | 8 +- libpathod/pathoc_cmdline.py | 15 ++- libpathod/pathod.py | 214 +++++++++++++++++++++++---------- libpathod/pathod_cmdline.py | 15 ++- libpathod/utils.py | 10 +- test/test_language_http2.py | 3 +- test/test_pathoc.py | 7 +- 10 files changed, 297 insertions(+), 110 deletions(-) diff --git a/libpathod/language/__init__.py b/libpathod/language/__init__.py index ae9a8c764..10050bf82 100644 --- a/libpathod/language/__init__.py +++ b/libpathod/language/__init__.py @@ -19,7 +19,7 @@ def expand(msg): yield msg -def parse_pathod(s): +def parse_pathod(s, use_http2=False): """ May raise ParseException """ @@ -28,12 +28,17 @@ def parse_pathod(s): except UnicodeError: raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0) try: - reqs = pp.Or( - [ + if use_http2: + expressions = [ + # http2.Frame.expr(), + http2.Response.expr(), + ] + else: + expressions = [ websockets.WebsocketFrame.expr(), http.Response.expr(), ] - ).parseString(s, parseAll=True) + reqs = pp.Or(expressions).parseString(s, parseAll=True) except pp.ParseException as v: raise exceptions.ParseException(v.msg, v.line, v.col) return itertools.chain(*[expand(i) for i in reqs]) @@ -55,7 +60,6 @@ def parse_pathoc(s, use_http2=False): websockets.WebsocketClientFrame.expr(), http.Request.expr(), ] - reqs = pp.OneOrMore(pp.Or(expressions)).parseString(s, parseAll=True) except pp.ParseException as v: raise exceptions.ParseException(v.msg, v.line, v.col) diff --git a/libpathod/language/http.py b/libpathod/language/http.py index 9a8404f04..115f8069b 100644 --- a/libpathod/language/http.py +++ b/libpathod/language/http.py @@ -367,10 +367,6 @@ class Request(_HTTPMessage): return ":".join([i.spec() for i in self.tokens]) -class PathodErrorResponse(Response): - pass - - def make_error_response(reason, body=None): tokens = [ Code("800"), @@ -381,4 +377,4 @@ def make_error_response(reason, body=None): Reason(base.TokValueLiteral(reason)), Body(base.TokValueLiteral("pathod error: " + (body or reason))), ] - return PathodErrorResponse(tokens) + return Response(tokens) diff --git a/libpathod/language/http2.py b/libpathod/language/http2.py index 4a5b90840..1d2517d37 100644 --- a/libpathod/language/http2.py +++ b/libpathod/language/http2.py @@ -45,14 +45,20 @@ class Body(base.Value): class Times(base.Integer): preamble = "x" +class Code(base.Integer): + pass class Request(message.Message): comps = ( Header, Body, - Times, ) + logattrs = ["method", "path"] + + def __init__(self, tokens): + super(Response, self).__init__(tokens) + self.rendered_values = None @property def method(self): @@ -87,7 +93,6 @@ class Request(message.Message): Method.expr(), base.Sep, Path.expr(), - base.Sep, pp.ZeroOrMore(base.Sep + atom) ] ) @@ -95,21 +100,109 @@ class Request(message.Message): return resp def resolve(self, settings, msg=None): - tokens = self.tokens[:] - return self.__class__( - [i.resolve(settings, self) for i in tokens] - ) + return self def values(self, settings): - return settings.protocol.create_request( - self.method.string(), - self.path, - self.headers, - self.body) + if self.rendered_values: + return self.rendered_values + else: + headers = self.headers + if headers: + headers = headers.values(settings) + + body = self.body + if body: + body = body.string() + + self.rendered_values = settings.protocol.create_request( + self.method.string(), + self.path.string(), + headers, # TODO: parse that into a dict?! + body) + return self.rendered_values def spec(self): return ":".join([i.spec() for i in self.tokens]) +class Response(message.Message): + unique_name = None + comps = ( + Header, + Body, + ) + + def __init__(self, tokens): + super(Response, self).__init__(tokens) + self.rendered_values = None + self.stream_id = 0 + + @property + def code(self): + return self.tok(Code) + + @property + def headers(self): + return self.toks(Header) + + @property + def body(self): + return self.tok(Body) + + @property + def actions(self): + return [] + + def resolve(self, settings, msg=None): + return self + + @classmethod + def expr(klass): + parts = [i.expr() for i in klass.comps] + atom = pp.MatchFirst(parts) + resp = pp.And( + [ + Code.expr(), + pp.ZeroOrMore(base.Sep + atom) + ] + ) + resp = resp.setParseAction(klass) + return resp + + def values(self, settings): + if self.rendered_values: + return self.rendered_values + else: + headers = self.headers + if headers: + headers = headers.values(settings) + + body = self.body + if body: + body = body.values(settings) + + self.rendered_values = settings.protocol.create_response( + self.code.string(), + self.stream_id, + headers, # TODO: parse that into a dict?! + body) + return self.rendered_values + + def spec(self): + return ":".join([i.spec() for i in self.tokens]) + +def make_error_response(reason, body=None): + raise NotImplementedError + # tokens = [ + # Code("800"), + # Header( + # base.TokValueLiteral("Content-Type"), + # base.TokValueLiteral("text/plain") + # ), + # Reason(base.TokValueLiteral(reason)), + # Body(base.TokValueLiteral("pathod error: " + (body or reason))), + # ] + # return Response(tokens) + # class Frame(message.Message): # pass diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index e9bd5f569..9c0213600 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -155,13 +155,14 @@ class Pathoc(tcp.TCPClient): # SSL ssl=None, sni=None, - sslversion=4, + sslversion='SSLv23', clientcert=None, ciphers=None, # HTTP/2 use_http2=False, http2_skip_connection_preface=False, + http2_framedump = False, # Websockets ws_read_limit = None, @@ -199,6 +200,7 @@ class Pathoc(tcp.TCPClient): self.use_http2 = use_http2 self.http2_skip_connection_preface = http2_skip_connection_preface + self.http2_framedump = http2_framedump self.ws_read_limit = ws_read_limit @@ -219,7 +221,6 @@ class Pathoc(tcp.TCPClient): if not OpenSSL._util.lib.Cryptography_HAS_ALPN: # pragma: nocover print >> sys.stderr, "HTTP/2 requires ALPN support. Please use OpenSSL >= 1.0.2." print >> sys.stderr, "Pathoc might not be working as expected without ALPN." - self.protocol = http2.HTTP2Protocol(self) else: # TODO: create HTTP or Websockets protocol @@ -298,7 +299,7 @@ class Pathoc(tcp.TCPClient): if self.use_http2: self.protocol.check_alpn() if not self.http2_skip_connection_preface: - self.protocol.perform_connection_preface() + self.protocol.perform_client_connection_preface() if self.timeout: self.settimeout(self.timeout) @@ -466,6 +467,7 @@ def main(args): # pragma: nocover ciphers = args.ciphers, use_http2 = args.use_http2, http2_skip_connection_preface = args.http2_skip_connection_preface, + http2_framedump = args.http2_framedump, showreq = args.showreq, showresp = args.showresp, explain = args.explain, diff --git a/libpathod/pathoc_cmdline.py b/libpathod/pathoc_cmdline.py index dcd75d117..1d0df3b55 100644 --- a/libpathod/pathoc_cmdline.py +++ b/libpathod/pathoc_cmdline.py @@ -109,12 +109,11 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): help="SSL cipher specification" ) group.add_argument( - "--sslversion", dest="sslversion", type=int, default=4, - choices=[1, 2, 3, 4], - help=""" - Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default - to SSLv23. - """ + "--sslversion", dest="sslversion", type=str, default='SSLv23', + choices=utils.SSLVERSIONS.keys(), + help="""" + Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. + Default to SSLv23.""" ) group = parser.add_argument_group( @@ -156,6 +155,10 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): "-x", dest="hexdump", action="store_true", default=False, help="Output in hexdump format" ) + group.add_argument( + "--http2-framedump", dest="http2_framedump", action="store_true", default=False, + help="Output all received & sent HTTP/2 frames" + ) args = parser.parse_args(argv[1:]) diff --git a/libpathod/pathod.py b/libpathod/pathod.py index f14961810..535340cbb 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -7,7 +7,7 @@ import urllib import re import time -from netlib import tcp, http, wsgi, certutils, websockets +from netlib import tcp, http, http2, wsgi, certutils, websockets, odict from . import version, app, language, utils, log import language.http @@ -40,7 +40,7 @@ class SSLOptions: sslversion=tcp.SSLv23_METHOD, ciphers=None, certs=None, - alpn_select=None, + alpn_select=http2.HTTP2Protocol.ALPN_PROTO_H2, ): self.confdir = confdir self.cn = cn @@ -69,32 +69,37 @@ class PathodHandler(tcp.BaseHandler): wbufsize = 0 sni = None - def __init__(self, connection, address, server, logfp, settings): - self.logfp = logfp + def __init__(self, connection, address, server, logfp, settings, http2_framedump=False): tcp.BaseHandler.__init__(self, connection, address, server) + self.logfp = logfp self.settings = copy.copy(settings) + self.protocol = None + self.use_http2 = False + self.http2_framedump = http2_framedump - def handle_sni(self, connection): + def _handle_sni(self, connection): self.sni = connection.get_servername() def http_serve_crafted(self, crafted): + """ + This method is HTTP/1 and HTTP/2 capable. + """ + error, crafted = self.server.check_policy( crafted, self.settings ) if error: - err = language.http.make_error_response(error) + err = self.make_http_error_response(error) language.serve(err, self.wfile, self.settings) return None, dict( type="error", msg = error ) - if self.server.explain and not isinstance( - crafted, - language.http.PathodErrorResponse - ): + if self.server.explain and not hasattr(crafted, 'is_error_response'): crafted = crafted.freeze(self.settings) log.write(self.logfp, ">> Spec: %s" % crafted.spec()) + response_log = language.serve( crafted, self.wfile, @@ -154,6 +159,8 @@ class PathodHandler(tcp.BaseHandler): def handle_http_connect(self, connect, lg): """ + This method is HTTP/1 only. + Handle a CONNECT request. """ http.read_headers(self.rfile) @@ -171,7 +178,7 @@ class PathodHandler(tcp.BaseHandler): self.convert_to_ssl( cert, key, - handle_sni=self.handle_sni, + handle_sni=self._handle_sni, request_client_cert=self.server.ssloptions.request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.sslversion, @@ -185,10 +192,12 @@ class PathodHandler(tcp.BaseHandler): def handle_http_app(self, method, path, headers, content, lg): """ + This method is HTTP/1 only. + Handle a request to the built-in app. """ if self.server.noweb: - crafted = language.http.make_error_response("Access Denied") + crafted = self.make_http_error_response("Access Denied") language.serve(crafted, self.wfile, self.settings) return None, dict( type="error", @@ -209,6 +218,8 @@ class PathodHandler(tcp.BaseHandler): def handle_http_request(self): """ + This method is HTTP/1 and HTTP/2 capable. + Returns a (handler, log) tuple. handler: Handler for the next request, or None to disconnect @@ -217,28 +228,24 @@ class PathodHandler(tcp.BaseHandler): lr = self.rfile if self.server.logreq else None lw = self.wfile if self.server.logresp else None with log.Log(self.logfp, self.server.hexdump, lr, lw) as lg: - line = http.get_request_line(self.rfile) - if not line: - # Normal termination - return None, None - - m = utils.MemBool() - if m(http.parse_init_connect(line)): - return self.handle_http_connect(m.v, lg) - elif m(http.parse_init_proxy(line)): - method, _, _, _, path, httpversion = m.v - elif m(http.parse_init_http(line)): - method, path, httpversion = m.v + if self.use_http2: + self.protocol.perform_server_connection_preface() + stream_id, headers, body = self.protocol.read_request() + method = headers[':method'] + path = headers[':path'] + headers = odict.ODict(headers) else: - s = "Invalid first line: %s" % repr(line) - lg(s) - return None, dict(type="error", msg=s) - - headers = http.read_headers(self.rfile) - if headers is None: - s = "Invalid headers" - lg(s) - return None, dict(type="error", msg=s) + req = self.read_http_request(lg) + if 'next_handle' in req: + return req['next_handle'] + if 'errors' in req: + return None, req['errors'] + if not 'method' in req or not 'path' in req: + return None, None + method = req['method'] + path = req['path'] + headers = req['headers'] + body = req['body'] clientcert = None if self.clientcert: @@ -258,7 +265,7 @@ class PathodHandler(tcp.BaseHandler): path=path, method=method, headers=headers.lst, - httpversion=httpversion, + # httpversion=httpversion, sni=self.sni, remote_address=self.address(), clientcert=clientcert, @@ -268,16 +275,6 @@ class PathodHandler(tcp.BaseHandler): if self.ssl_established: retlog["cipher"] = self.get_current_cipher() - try: - content = http.read_http_body( - self.rfile, headers, None, - method, None, True - ) - except http.HttpError as s: - s = str(s) - lg(s) - return None, dict(type="error", msg=s) - m = utils.MemBool() websocket_key = websockets.check_client_handshake(headers) self.settings.websocket_key = websocket_key @@ -288,28 +285,37 @@ class PathodHandler(tcp.BaseHandler): anchor_gen = language.parse_pathod("ws") else: anchor_gen = None - for i in self.server.anchors: - if i[0].match(path): - anchor_gen = i[1] + + for regex, spec in self.server.anchors: + if regex.match(path): + anchor_gen = language.parse_pathod(spec, self.use_http2) break else: - print(self.server.craftanchor) if m(path.startswith(self.server.craftanchor)): spec = urllib.unquote(path)[len(self.server.craftanchor):] if spec: try: - anchor_gen = language.parse_pathod(spec) + anchor_gen = language.parse_pathod(spec, self.use_http2) except language.ParseException as v: lg("Parse error: %s" % v.msg) - anchor_gen = iter([language.http.make_error_response( + anchor_gen = iter([self.make_http_error_response( "Parse Error", "Error parsing response spec: %s\n" % ( v.msg + v.marked() ) )]) + else: + if self.use_http2: + raise NotImplementedError(\ + "HTTP/2 only supports request/response with the craft anchor point.") + if anchor_gen: spec = anchor_gen.next() + + if self.use_http2 and isinstance(spec, language.http2.Response): + spec.stream_id = stream_id + lg("crafting spec: %s" % spec) nexthandler, retlog["response"] = self.http_serve_crafted( spec @@ -319,28 +325,77 @@ class PathodHandler(tcp.BaseHandler): else: return nexthandler, retlog else: - return self.handle_http_app(method, path, headers, content, lg) + return self.handle_http_app(method, path, headers, body, lg) - def addlog(self, log): - # FIXME: The bytes in the log should not be escaped. We do this at the - # moment because JSON encoding can't handle binary data, and I don't - # want to base64 everything. - if self.server.logreq: - bytes = self.rfile.get_log().encode("string_escape") - log["request_bytes"] = bytes - if self.server.logresp: - bytes = self.wfile.get_log().encode("string_escape") - log["response_bytes"] = bytes - self.server.add_log(log) + def read_http_request(self, lg): + """ + This method is HTTP/1 only. + """ + line = http.get_request_line(self.rfile) + if not line: + # Normal termination + return dict() + + m = utils.MemBool() + if m(http.parse_init_connect(line)): + return dict(next_handle=self.handle_http_connect(m.v, lg)) + elif m(http.parse_init_proxy(line)): + method, _, _, _, path, httpversion = m.v + elif m(http.parse_init_http(line)): + method, path, httpversion = m.v + else: + s = "Invalid first line: %s" % repr(line) + lg(s) + return dict(errors=dict(type="error", msg=s)) + + headers = http.read_headers(self.rfile) + if headers is None: + s = "Invalid headers" + lg(s) + return dict(errors=dict(type="error", msg=s)) + + try: + body = http.read_http_body( + self.rfile, + headers, + None, + method, + None, + True, + ) + except http.HttpError as s: + s = str(s) + lg(s) + return dict(errors=dict(type="error", msg=s)) + + return dict( + method=method, + path=path, + headers=headers, + body=body) + + def make_http_error_response(self, reason, body=None): + """ + This method is HTTP/1 and HTTP/2 capable. + """ + if self.use_http2: + resp = language.http2.make_error_response(reason, body) + else: + resp = language.http.make_error_response(reason, body) + resp.is_error_response = True + return resp def handle(self): + self.settimeout(self.server.timeout) + if self.server.ssl: try: cert, key, _ = self.server.ssloptions.get_cert(None) self.convert_to_ssl( cert, key, - handle_sni=self.handle_sni, + dhparams=self.server.ssloptions.certstore.dhparams, + handle_sni=self._handle_sni, request_client_cert=self.server.ssloptions.request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.sslversion, @@ -356,8 +411,20 @@ class PathodHandler(tcp.BaseHandler): ) log.write(self.logfp, s) return - self.settimeout(self.server.timeout) + + alp = self.get_alpn_proto_negotiated() + if alp == http2.HTTP2Protocol.ALPN_PROTO_H2: + self.protocol = http2.HTTP2Protocol(self, is_server=True, dump_frames=self.http2_framedump) + self.use_http2 = True + + # if not self.protocol: + # # TODO: create HTTP or Websockets protocol + # self.protocol = None + + self.settings.protocol = self.protocol + handler = self.handle_http_request + while not self.finished: handler, l = handler() if l: @@ -365,6 +432,18 @@ class PathodHandler(tcp.BaseHandler): if not handler: return + def addlog(self, log): + # FIXME: The bytes in the log should not be escaped. We do this at the + # moment because JSON encoding can't handle binary data, and I don't + # want to base64 everything. + if self.server.logreq: + bytes = self.rfile.get_log().encode("string_escape") + log["request_bytes"] = bytes + if self.server.logresp: + bytes = self.wfile.get_log().encode("string_escape") + log["response_bytes"] = bytes + self.server.add_log(log) + class Pathod(tcp.TCPServer): LOGBUF = 500 @@ -387,6 +466,7 @@ class Pathod(tcp.TCPServer): logresp=False, explain=False, hexdump=False, + http2_framedump=False, webdebug=False, logfp=sys.stdout, ): @@ -414,6 +494,7 @@ class Pathod(tcp.TCPServer): self.noapi, self.nohang = noapi, nohang self.timeout, self.logreq = timeout, logreq self.logresp, self.hexdump = logresp, hexdump + self.http2_framedump = http2_framedump self.explain = explain self.logfp = logfp @@ -451,7 +532,8 @@ class Pathod(tcp.TCPServer): client_address, self, self.logfp, - self.settings + self.settings, + self.http2_framedump, ) try: h.handle() @@ -508,7 +590,6 @@ def main(args): # pragma: nocover sslversion = utils.SSLVERSIONS[args.sslversion], certs = args.ssl_certs, sans = args.sans, - alpn_select = args.alpn_select, ) root = logging.getLogger() @@ -548,6 +629,7 @@ def main(args): # pragma: nocover logreq = args.logreq, logresp = args.logresp, hexdump = args.hexdump, + http2_framedump = args.http2_framedump, explain = args.explain, webdebug = args.webdebug ) diff --git a/libpathod/pathod_cmdline.py b/libpathod/pathod_cmdline.py index f1bb6982a..4343401ff 100644 --- a/libpathod/pathod_cmdline.py +++ b/libpathod/pathod_cmdline.py @@ -139,10 +139,11 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr): """ ) group.add_argument( - "--sslversion", dest="sslversion", type=int, default=4, - choices=[1, 2, 3, 4], - help=""""Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default - to SSLv23.""" + "--sslversion", dest="sslversion", type=str, default='SSLv23', + choices=utils.SSLVERSIONS.keys(), + help="""" + Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. + Default to SSLv23.""" ) group = parser.add_argument_group( @@ -172,6 +173,12 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr): "-x", dest="hexdump", action="store_true", default=False, help="Log request/response in hexdump format" ) + group.add_argument( + "--http2-framedump", dest="http2_framedump", action="store_true", default=False, + help="Output all received & sent HTTP/2 frames" + ) + + args = parser.parse_args(argv[1:]) certs = [] diff --git a/libpathod/utils.py b/libpathod/utils.py index 9bd2812e5..481c51372 100644 --- a/libpathod/utils.py +++ b/libpathod/utils.py @@ -3,10 +3,12 @@ import sys from netlib import tcp SSLVERSIONS = { - 1: tcp.TLSv1_METHOD, - 2: tcp.SSLv2_METHOD, - 3: tcp.SSLv3_METHOD, - 4: tcp.SSLv23_METHOD, + 'TLSv1.2': tcp.TLSv1_2_METHOD, + 'TLSv1.1': tcp.TLSv1_1_METHOD, + 'TLSv1': tcp.TLSv1_METHOD, + 'SSLv3': tcp.SSLv3_METHOD, + 'SSLv2': tcp.SSLv2_METHOD, + 'SSLv23': tcp.SSLv23_METHOD, } SIZE_UNITS = dict( diff --git a/test/test_language_http2.py b/test/test_language_http2.py index de3e5cf93..3c751fd1d 100644 --- a/test/test_language_http2.py +++ b/test/test_language_http2.py @@ -1,5 +1,6 @@ import cStringIO +from netlib import tcp from libpathod import language from libpathod.language import http2, base import netlib @@ -64,7 +65,7 @@ class TestRequest: s, language.Settings( request_host = "foo.com", - protocol = netlib.http2.HTTP2Protocol(None) + protocol = netlib.http2.HTTP2Protocol(tcp.TCPClient(('localhost', 1234))) ) ) diff --git a/test/test_pathoc.py b/test/test_pathoc.py index e1e1fe97b..d39f92759 100644 --- a/test/test_pathoc.py +++ b/test/test_pathoc.py @@ -230,9 +230,6 @@ class TestDaemon(_TestDaemon): class TestDaemonHTTP2(_TestDaemon): ssl = True - ssloptions = pathod.SSLOptions( - alpn_select = http2.HTTP2Protocol.ALPN_PROTO_H2, - ) def test_http2(self): c = pathoc.Pathoc( @@ -270,5 +267,5 @@ class TestDaemonHTTP2(_TestDaemon): use_http2 = True, ) c.connect() - resp = c.request("get:/api/info") - assert tuple(json.loads(resp.content)["version"]) == version.IVERSION + resp = c.request("get:/p/200") + assert resp.status_code == "200" From ec68aa303e89398ba34bbe01f3fbd1ac1fc441f3 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 16 Jun 2015 11:33:10 +0200 Subject: [PATCH 09/10] http2: implement error response --- libpathod/language/http2.py | 18 ++++++------------ libpathod/pathod.py | 7 +++++-- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/libpathod/language/http2.py b/libpathod/language/http2.py index 1d2517d37..c28b904ea 100644 --- a/libpathod/language/http2.py +++ b/libpathod/language/http2.py @@ -179,7 +179,7 @@ class Response(message.Message): body = self.body if body: - body = body.values(settings) + body = body.string() self.rendered_values = settings.protocol.create_response( self.code.string(), @@ -192,17 +192,11 @@ class Response(message.Message): return ":".join([i.spec() for i in self.tokens]) def make_error_response(reason, body=None): - raise NotImplementedError - # tokens = [ - # Code("800"), - # Header( - # base.TokValueLiteral("Content-Type"), - # base.TokValueLiteral("text/plain") - # ), - # Reason(base.TokValueLiteral(reason)), - # Body(base.TokValueLiteral("pathod error: " + (body or reason))), - # ] - # return Response(tokens) + tokens = [ + Code("800"), + Body(base.TokValueLiteral("pathod error: " + (body or reason))), + ] + return Response(tokens) # class Frame(message.Message): # pass diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 535340cbb..b6f04b923 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -306,8 +306,11 @@ class PathodHandler(tcp.BaseHandler): )]) else: if self.use_http2: - raise NotImplementedError(\ - "HTTP/2 only supports request/response with the craft anchor point.") + anchor_gen = iter([self.make_http_error_response( + "Spec Error", + "HTTP/2 only supports request/response with the craft anchor point: %s" % + self.server.craftanchor + )]) if anchor_gen: From 408b4ffef0a784bea7ec08c252e757bca6e28134 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 16 Jun 2015 13:52:41 +0200 Subject: [PATCH 10/10] http2: implement Headers for request & response improve test coverage fix super ctor call fix legacy httpversion simpliy SSLInfo without ALPN --- libpathod/language/http2.py | 19 +++--- libpathod/pathoc.py | 7 +-- libpathod/pathod.py | 7 ++- test/test_language_http2.py | 111 ++++++++++++++++++++++++++++++++++-- test/test_pathod.py | 14 ++++- test/tutils.py | 6 +- 6 files changed, 141 insertions(+), 23 deletions(-) diff --git a/libpathod/language/http2.py b/libpathod/language/http2.py index c28b904ea..dec2d5fef 100644 --- a/libpathod/language/http2.py +++ b/libpathod/language/http2.py @@ -35,8 +35,15 @@ class Path(base.Value): class Header(base.KeyValue): + unique_name = None preamble = "h" + def values(self, settings): + return ( + self.key.get_generator(settings), + self.value.get_generator(settings), + ) + class Body(base.Value): preamble = "b" @@ -45,9 +52,11 @@ class Body(base.Value): class Times(base.Integer): preamble = "x" + class Code(base.Integer): pass + class Request(message.Message): comps = ( Header, @@ -57,7 +66,7 @@ class Request(message.Message): logattrs = ["method", "path"] def __init__(self, tokens): - super(Response, self).__init__(tokens) + super(Request, self).__init__(tokens) self.rendered_values = None @property @@ -106,9 +115,7 @@ class Request(message.Message): if self.rendered_values: return self.rendered_values else: - headers = self.headers - if headers: - headers = headers.values(settings) + headers = [header.values(settings) for header in self.headers] body = self.body if body: @@ -173,9 +180,7 @@ class Response(message.Message): if self.rendered_values: return self.rendered_values else: - headers = self.headers - if headers: - headers = headers.values(settings) + headers = [header.values(settings) for header in self.headers] body = self.body if body: diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index 9c0213600..c42cc82af 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -30,13 +30,8 @@ class SSLInfo: self.certchain, self.cipher, self.alp = certchain, cipher, alp def __str__(self): - if self.alp: - alp = self.alp - else: - alp = '' - parts = [ - "Application Layer Protocol: %s" % alp, + "Application Layer Protocol: %s" % self.alp, "Cipher: %s, %s bit, %s" % self.cipher, "SSL certificate chain:" ] diff --git a/libpathod/pathod.py b/libpathod/pathod.py index b6f04b923..212abbdcc 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -234,6 +234,7 @@ class PathodHandler(tcp.BaseHandler): method = headers[':method'] path = headers[':path'] headers = odict.ODict(headers) + httpversion = "" else: req = self.read_http_request(lg) if 'next_handle' in req: @@ -246,6 +247,7 @@ class PathodHandler(tcp.BaseHandler): path = req['path'] headers = req['headers'] body = req['body'] + httpversion = req['httpversion'] clientcert = None if self.clientcert: @@ -265,7 +267,7 @@ class PathodHandler(tcp.BaseHandler): path=path, method=method, headers=headers.lst, - # httpversion=httpversion, + httpversion=httpversion, sni=self.sni, remote_address=self.address(), clientcert=clientcert, @@ -375,7 +377,8 @@ class PathodHandler(tcp.BaseHandler): method=method, path=path, headers=headers, - body=body) + body=body, + httpversion=httpversion) def make_http_error_response(self, reason, body=None): """ diff --git a/test/test_language_http2.py b/test/test_language_http2.py index 3c751fd1d..0be42253c 100644 --- a/test/test_language_http2.py +++ b/test/test_language_http2.py @@ -10,8 +10,29 @@ import tutils def parse_request(s): return language.parse_pathoc(s, True).next() +def parse_response(s): + return language.parse_pathod(s, True).next() + +def default_settings(): + return language.Settings( + request_host = "foo.com", + protocol = netlib.http2.HTTP2Protocol(tcp.TCPClient(('localhost', 1234))) + ) + + +def test_make_error_response(): + d = cStringIO.StringIO() + s = http2.make_error_response("foo", "bar") + language.serve(s, d, default_settings()) + class TestRequest: + def test_cached_values(self): + req = parse_request("get:/") + req_id = id(req) + assert req_id == id(req.resolve(default_settings())) + assert req.values(default_settings()) == req.values(default_settings()) + def test_nonascii(self): tutils.raises("ascii", parse_request, "get:\xf0") @@ -57,16 +78,31 @@ class TestRequest: assert r[0].method.string() == "GET" assert r[1].method.string() == "GET" - def test_render(self): + def test_render_simple(self): s = cStringIO.StringIO() r = parse_request("GET:'/foo'") assert language.serve( r, s, - language.Settings( - request_host = "foo.com", - protocol = netlib.http2.HTTP2Protocol(tcp.TCPClient(('localhost', 1234))) - ) + default_settings(), + ) + + def test_render_with_headers(self): + s = cStringIO.StringIO() + r = parse_request('GET:/foo:h"foo"="bar"') + assert language.serve( + r, + s, + default_settings(), + ) + + def test_render_with_body(self): + s = cStringIO.StringIO() + r = parse_request("GET:'/foo':bfoobar") + assert language.serve( + r, + s, + default_settings(), ) def test_spec(self): @@ -74,3 +110,68 @@ class TestRequest: s = parse_request(s).spec() assert parse_request(s).spec() == s rt("get:/foo") + + +class TestResponse: + def test_cached_values(self): + res = parse_response("200") + res_id = id(res) + assert res_id == id(res.resolve(default_settings())) + assert res.values(default_settings()) == res.values(default_settings()) + + def test_nonascii(self): + tutils.raises("ascii", parse_response, "200:\xf0") + + def test_err(self): + tutils.raises(language.ParseException, parse_response, 'GET:/') + + def test_simple(self): + r = parse_response('200') + assert r.code.string() == "200" + assert len(r.headers) == 0 + + r = parse_response('200:h"foo"="bar"') + assert r.code.string() == "200" + assert len(r.headers) == 1 + assert r.headers[0].values(default_settings()) == ("foo", "bar") + assert r.body == None + + r = parse_response('200:h"foo"="bar":bfoobar:h"bla"="fasel"') + assert r.code.string() == "200" + assert len(r.headers) == 2 + assert r.headers[0].values(default_settings()) == ("foo", "bar") + assert r.headers[1].values(default_settings()) == ("bla", "fasel") + assert r.body.string() == "foobar" + + def test_render_simple(self): + s = cStringIO.StringIO() + r = parse_response('200') + assert language.serve( + r, + s, + default_settings(), + ) + + def test_render_with_headers(self): + s = cStringIO.StringIO() + r = parse_response('200:h"foo"="bar"') + assert language.serve( + r, + s, + default_settings(), + ) + + def test_render_with_body(self): + s = cStringIO.StringIO() + r = parse_response('200:bfoobar') + assert language.serve( + r, + s, + default_settings(), + ) + + def test_spec(self): + def rt(s): + s = parse_response(s).spec() + assert parse_response(s).spec() == s + rt("200:bfoobar") diff --git a/test/test_pathod.py b/test/test_pathod.py index f85ef38d0..1a3a5004d 100644 --- a/test/test_pathod.py +++ b/test/test_pathod.py @@ -1,7 +1,7 @@ import sys import cStringIO from libpathod import pathod, version -from netlib import tcp, http +from netlib import tcp, http, http2 import tutils @@ -269,3 +269,15 @@ class TestDaemonSSL(CommonTests): r, _ = self.pathoc([r"get:/p/202"]) assert r[0].status_code == 202 assert self.d.last_log()["cipher"][1] > 0 + +class TestHTTP2(tutils.DaemonTests): + force_http2 = True + ssl = True + noweb = True + noapi = True + nohang = True + + def test_http2(self): + r, _ = self.pathoc(["GET:/"], ssl=True, use_http2=True) + print(r) + assert r[0].status_code == "800" diff --git a/test/tutils.py b/test/tutils.py index 60c0765a5..2184ade56 100644 --- a/test/tutils.py +++ b/test/tutils.py @@ -73,7 +73,8 @@ class DaemonTests(object): timeout=None, connect_to=None, ssl=None, - ws_read_limit=None + ws_read_limit=None, + use_http2=False, ): """ Returns a (messages, text log) tuple. @@ -86,7 +87,8 @@ class DaemonTests(object): ssl=ssl, ws_read_limit=ws_read_limit, timeout = timeout, - fp = logfp + fp = logfp, + use_http2 = use_http2, ) c.connect(connect_to) ret = []