diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py deleted file mode 100644 index 06a6c533c..000000000 --- a/libpathod/cmdline.py +++ /dev/null @@ -1,442 +0,0 @@ -#!/usr/bin/env python -import sys -import argparse -import os -import os.path -import re -from netlib import http_uastrings -from . import pathoc, pathod, version, utils, language - - -def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): - preparser = argparse.ArgumentParser(add_help=False) - preparser.add_argument( - "--show-uas", dest="showua", action="store_true", default=False, - help="Print user agent shortcuts and exit." - ) - pa = preparser.parse_known_args(argv)[0] - if pa.showua: - print >> stdout, "User agent strings:" - for i in http_uastrings.UASTRINGS: - print >> stdout, " ", i[1], i[0] - sys.exit(0) - - parser = argparse.ArgumentParser( - description='A perverse HTTP client.', parents=[preparser] - ) - parser.add_argument( - '--version', - action='version', - version="pathoc " + version.VERSION - ) - parser.add_argument( - "-c", dest="connect_to", type=str, default=False, - metavar = "HOST:PORT", - help="Issue an HTTP CONNECT to connect to the specified host." - ) - parser.add_argument( - "--memo-limit", dest='memolimit', default=5000, type=int, metavar="N", - help='Stop if we do not find a valid request after N attempts.' - ) - parser.add_argument( - "-m", dest='memo', action="store_true", default=False, - help=""" - Remember specs, and never play the same one twice. Note that this - means requests have to be rendered in memory, which means that - large generated data can cause issues. - """ - ) - parser.add_argument( - "-n", dest='repeat', default=1, type=int, metavar="N", - help='Repeat N times. If 0 repeat for ever.' - ) - parser.add_argument( - "-w", dest='wait', default=0, type=float, metavar="N", - help='Wait N seconds between each request.' - ) - parser.add_argument( - "-r", dest="random", action="store_true", default=False, - help=""" - Select a random request from those specified. If this is not specified, - requests are all played in sequence. - """ - ) - parser.add_argument( - "-t", dest="timeout", type=int, default=None, - help="Connection timeout" - ) - parser.add_argument( - "--http2", dest="use_http2", action="store_true", default=False, - help='Perform all requests over a single HTTP/2 connection.' - ) - parser.add_argument( - "--http2-skip-connection-preface", - dest="http2_skip_connection_preface", - action="store_true", - default=False, - help='Skips the HTTP/2 connection preface before sending requests.') - - parser.add_argument( - 'host', type=str, - metavar = "host[:port]", - help='Host and port to connect to' - ) - parser.add_argument( - 'requests', type=str, nargs="+", - help=""" - Request specification, or path to a file containing request - specifcations - """ - ) - - group = parser.add_argument_group( - 'SSL', - ) - group.add_argument( - "-s", dest="ssl", action="store_true", default=False, - help="Connect with SSL" - ) - group.add_argument( - "-C", dest="clientcert", type=str, default=False, - help="Path to a file containing client certificate and private key" - ) - group.add_argument( - "-i", dest="sni", type=str, default=False, - help="SSL Server Name Indication" - ) - group.add_argument( - "--ciphers", dest="ciphers", type=str, default=False, - help="SSL cipher specification" - ) - group.add_argument( - "--sslversion", dest="sslversion", type=int, default=4, - choices=[1, 2, 3, 4], - help=""" - Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default - to SSLv23. - """ - ) - - group = parser.add_argument_group( - 'Controlling Output', - """ - Some of these options expand generated values for logging - if - you're generating large data, use them with caution. - """ - ) - group.add_argument( - "-I", dest="ignorecodes", type=str, default="", - help="Comma-separated list of response codes to ignore" - ) - group.add_argument( - "-S", dest="showssl", action="store_true", default=False, - help="Show info on SSL connection" - ) - group.add_argument( - "-e", dest="explain", action="store_true", default=False, - help="Explain requests" - ) - group.add_argument( - "-o", dest="oneshot", action="store_true", default=False, - help="Oneshot - exit after first non-ignored response" - ) - group.add_argument( - "-q", dest="showreq", action="store_true", default=False, - help="Print full request" - ) - group.add_argument( - "-p", dest="showresp", action="store_true", default=False, - help="Print full response" - ) - group.add_argument( - "-T", dest="ignoretimeout", action="store_true", default=False, - help="Ignore timeouts" - ) - group.add_argument( - "-x", dest="hexdump", action="store_true", default=False, - help="Output in hexdump format" - ) - - args = parser.parse_args(argv[1:]) - - args.port = None - if ":" in args.host: - h, p = args.host.rsplit(":", 1) - try: - p = int(p) - except ValueError: - return parser.error("Invalid port in host spec: %s" % args.host) - args.host = h - args.port = p - - if args.port is None: - args.port = 443 if args.ssl else 80 - - try: - args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i] - except ValueError: - return parser.error( - "Invalid return code specification: %s" % - args.ignorecodes) - - if args.connect_to: - parts = args.connect_to.split(":") - if len(parts) != 2: - return parser.error( - "Invalid CONNECT specification: %s" % - args.connect_to) - try: - parts[1] = int(parts[1]) - except ValueError: - return parser.error( - "Invalid CONNECT specification: %s" % - args.connect_to) - args.connect_to = parts - else: - args.connect_to = None - - reqs = [] - for r in args.requests: - if os.path.isfile(r): - data = open(r).read() - r = data - try: - reqs.append(language.parse_pathoc(r, args.use_http2)) - except language.ParseException as v: - print >> stderr, "Error parsing request spec: %s" % v.msg - print >> stderr, v.marked() - sys.exit(1) - args.requests = reqs - return args - - -def go_pathoc(): # pragma: nocover - args = args_pathoc(sys.argv) - pathoc.main(args) - - -def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr): - parser = argparse.ArgumentParser( - description='A pathological HTTP/S daemon.' - ) - parser.add_argument( - '--version', - action='version', - version="pathod " + version.VERSION - ) - parser.add_argument( - "-p", - dest='port', - default=9999, - type=int, - help='Port. Specify 0 to pick an arbitrary empty port. (9999)' - ) - parser.add_argument( - "-l", - dest='address', - default="127.0.0.1", - type=str, - help='Listening address. (127.0.0.1)' - ) - parser.add_argument( - "-a", - dest='anchors', - default=[], - type=str, - action="append", - metavar="ANCHOR", - help=""" - Add an anchor. Specified as a string with the form - pattern=spec or pattern=filepath, where pattern is a regular - expression. - """ - ) - parser.add_argument( - "-c", dest='craftanchor', default=pathod.DEFAULT_ANCHOR, type=str, - help=""" - Regular expression specifying anchor point for URL crafting - commands. (%s) - """%pathod.DEFAULT_ANCHOR - ) - parser.add_argument( - "--confdir", - action="store", type = str, dest="confdir", default='~/.mitmproxy', - help = "Configuration directory. (~/.mitmproxy)" - ) - parser.add_argument( - "-d", dest='staticdir', default=None, type=str, - help='Directory for static files.' - ) - parser.add_argument( - "-D", dest='daemonize', default=False, action="store_true", - help='Daemonize.' - ) - parser.add_argument( - "-t", dest="timeout", type=int, default=None, - help="Connection timeout" - ) - parser.add_argument( - "--limit-size", - dest='sizelimit', - default=None, - type=str, - help='Size limit of served responses. Understands size suffixes, i.e. 100k.') - parser.add_argument( - "--noapi", dest='noapi', default=False, action="store_true", - help='Disable API.' - ) - parser.add_argument( - "--nohang", dest='nohang', default=False, action="store_true", - help='Disable pauses during crafted response generation.' - ) - parser.add_argument( - "--noweb", dest='noweb', default=False, action="store_true", - help='Disable both web interface and API.' - ) - parser.add_argument( - "--nocraft", - dest='nocraft', - default=False, - action="store_true", - help='Disable response crafting. If anchors are specified, they still work.') - parser.add_argument( - "--webdebug", dest='webdebug', default=False, action="store_true", - help='Debugging mode for the web app (dev only).' - ) - - group = parser.add_argument_group( - 'SSL', - ) - group.add_argument( - "-s", dest='ssl', default=False, action="store_true", - help='Run in HTTPS mode.' - ) - group.add_argument( - "--cn", - dest="cn", - type=str, - default=None, - help="CN for generated SSL certs. Default: %s" % - pathod.DEFAULT_CERT_DOMAIN) - group.add_argument( - "-C", dest='ssl_not_after_connect', default=False, action="store_true", - help="Don't expect SSL after a CONNECT request." - ) - group.add_argument( - "--cert", dest='ssl_certs', default=[], type=str, - metavar = "SPEC", action="append", - help = """ - Add an SSL certificate. SPEC is of the form "[domain=]path". The domain - may include a wildcard, and is equal to "*" if not specified. The file - at path is a certificate in PEM format. If a private key is included in - the PEM, it is used, else the default key in the conf dir is used. Can - be passed multiple times. - """ - ) - group.add_argument( - "--ciphers", dest="ciphers", type=str, default=False, - help="SSL cipher specification" - ) - group.add_argument( - "--san", dest="sans", type=str, default=[], action="append", - metavar="SAN", - help=""" - Subject Altnernate Name to add to the server certificate. - May be passed multiple times. - """ - ) - group.add_argument( - "--sslversion", dest="sslversion", type=int, default=4, - choices=[1, 2, 3, 4], - help=""""Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default - to SSLv23.""" - ) - - group = parser.add_argument_group( - 'Controlling Logging', - """ - Some of these options expand generated values for logging - if - you're generating large data, use them with caution. - """ - ) - group.add_argument( - "-e", dest="explain", action="store_true", default=False, - help="Explain responses" - ) - group.add_argument( - "-f", dest='logfile', default=None, type=str, - help='Log to file.' - ) - group.add_argument( - "-q", dest="logreq", action="store_true", default=False, - help="Log full request" - ) - group.add_argument( - "-r", dest="logresp", action="store_true", default=False, - help="Log full response" - ) - group.add_argument( - "-x", dest="hexdump", action="store_true", default=False, - help="Log request/response in hexdump format" - ) - args = parser.parse_args(argv[1:]) - - certs = [] - for i in args.ssl_certs: - parts = i.split("=", 1) - if len(parts) == 1: - parts = ["*", parts[0]] - parts[1] = os.path.expanduser(parts[1]) - if not os.path.isfile(parts[1]): - return parser.error( - "Certificate file does not exist: %s" % - parts[1]) - certs.append(parts) - args.ssl_certs = certs - - alst = [] - for i in args.anchors: - parts = utils.parse_anchor_spec(i) - if not parts: - return parser.error("Invalid anchor specification: %s" % i) - alst.append(parts) - args.anchors = alst - - sizelimit = None - if args.sizelimit: - try: - sizelimit = utils.parse_size(args.sizelimit) - except ValueError as v: - return parser.error(v) - args.sizelimit = sizelimit - - try: - args.craftanchor = re.compile(args.craftanchor) - except re.error: - return parser.error( - "Invalid regex in craft anchor: %s" % args.craftanchor - ) - - anchors = [] - for patt, spec in args.anchors: - if os.path.isfile(spec): - data = open(spec).read() - spec = data - try: - req = language.parse_pathod(spec) - except language.ParseException as v: - print >> stderr, "Error parsing anchor spec: %s" % v.msg - print >> stderr, v.marked() - sys.exit(1) - try: - arex = re.compile(patt) - except re.error: - return parser.error("Invalid regex in anchor: %s" % patt) - anchors.append((arex, req)) - args.anchors = anchors - return args - - -def go_pathod(): # pragma: nocover - args = args_pathod(sys.argv) - pathod.main(args) diff --git a/libpathod/language/__init__.py b/libpathod/language/__init__.py index ae9a8c764..10050bf82 100644 --- a/libpathod/language/__init__.py +++ b/libpathod/language/__init__.py @@ -19,7 +19,7 @@ def expand(msg): yield msg -def parse_pathod(s): +def parse_pathod(s, use_http2=False): """ May raise ParseException """ @@ -28,12 +28,17 @@ def parse_pathod(s): except UnicodeError: raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0) try: - reqs = pp.Or( - [ + if use_http2: + expressions = [ + # http2.Frame.expr(), + http2.Response.expr(), + ] + else: + expressions = [ websockets.WebsocketFrame.expr(), http.Response.expr(), ] - ).parseString(s, parseAll=True) + reqs = pp.Or(expressions).parseString(s, parseAll=True) except pp.ParseException as v: raise exceptions.ParseException(v.msg, v.line, v.col) return itertools.chain(*[expand(i) for i in reqs]) @@ -55,7 +60,6 @@ def parse_pathoc(s, use_http2=False): websockets.WebsocketClientFrame.expr(), http.Request.expr(), ] - reqs = pp.OneOrMore(pp.Or(expressions)).parseString(s, parseAll=True) except pp.ParseException as v: raise exceptions.ParseException(v.msg, v.line, v.col) diff --git a/libpathod/language/http.py b/libpathod/language/http.py index 9a8404f04..115f8069b 100644 --- a/libpathod/language/http.py +++ b/libpathod/language/http.py @@ -367,10 +367,6 @@ class Request(_HTTPMessage): return ":".join([i.spec() for i in self.tokens]) -class PathodErrorResponse(Response): - pass - - def make_error_response(reason, body=None): tokens = [ Code("800"), @@ -381,4 +377,4 @@ def make_error_response(reason, body=None): Reason(base.TokValueLiteral(reason)), Body(base.TokValueLiteral("pathod error: " + (body or reason))), ] - return PathodErrorResponse(tokens) + return Response(tokens) diff --git a/libpathod/language/http2.py b/libpathod/language/http2.py index d78fc5c86..dec2d5fef 100644 --- a/libpathod/language/http2.py +++ b/libpathod/language/http2.py @@ -35,8 +35,15 @@ class Path(base.Value): class Header(base.KeyValue): + unique_name = None preamble = "h" + def values(self, settings): + return ( + self.key.get_generator(settings), + self.value.get_generator(settings), + ) + class Body(base.Value): preamble = "b" @@ -46,13 +53,21 @@ class Times(base.Integer): preamble = "x" +class Code(base.Integer): + pass + + class Request(message.Message): comps = ( Header, Body, - Times, ) + logattrs = ["method", "path"] + + def __init__(self, tokens): + super(Request, self).__init__(tokens) + self.rendered_values = None @property def method(self): @@ -87,7 +102,6 @@ class Request(message.Message): Method.expr(), base.Sep, Path.expr(), - base.Sep, pp.ZeroOrMore(base.Sep + atom) ] ) @@ -95,25 +109,99 @@ class Request(message.Message): return resp def resolve(self, settings, msg=None): - tokens = self.tokens[:] - return self.__class__( - [i.resolve(settings, self) for i in tokens] - ) + return self def values(self, settings): - return settings.protocol.create_request( - self.method.value.get_generator(settings), - self.path, - self.headers, - self.body) + if self.rendered_values: + return self.rendered_values + else: + headers = [header.values(settings) for header in self.headers] + + body = self.body + if body: + body = body.string() + + self.rendered_values = settings.protocol.create_request( + self.method.string(), + self.path.string(), + headers, # TODO: parse that into a dict?! + body) + return self.rendered_values def spec(self): return ":".join([i.spec() for i in self.tokens]) -# class H2F(base.CaselessLiteral): -# TOK = "h2f" -# -# -# class WebsocketFrame(message.Message): +class Response(message.Message): + unique_name = None + comps = ( + Header, + Body, + ) + + def __init__(self, tokens): + super(Response, self).__init__(tokens) + self.rendered_values = None + self.stream_id = 0 + + @property + def code(self): + return self.tok(Code) + + @property + def headers(self): + return self.toks(Header) + + @property + def body(self): + return self.tok(Body) + + @property + def actions(self): + return [] + + def resolve(self, settings, msg=None): + return self + + @classmethod + def expr(klass): + parts = [i.expr() for i in klass.comps] + atom = pp.MatchFirst(parts) + resp = pp.And( + [ + Code.expr(), + pp.ZeroOrMore(base.Sep + atom) + ] + ) + resp = resp.setParseAction(klass) + return resp + + def values(self, settings): + if self.rendered_values: + return self.rendered_values + else: + headers = [header.values(settings) for header in self.headers] + + body = self.body + if body: + body = body.string() + + self.rendered_values = settings.protocol.create_response( + self.code.string(), + self.stream_id, + headers, # TODO: parse that into a dict?! + body) + return self.rendered_values + + def spec(self): + return ":".join([i.spec() for i in self.tokens]) + +def make_error_response(reason, body=None): + tokens = [ + Code("800"), + Body(base.TokValueLiteral("pathod error: " + (body or reason))), + ] + return Response(tokens) + +# class Frame(message.Message): # pass diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index ba06b2f1a..c42cc82af 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -30,13 +30,8 @@ class SSLInfo: self.certchain, self.cipher, self.alp = certchain, cipher, alp def __str__(self): - if self.alp: - alp = self.alp - else: - alp = '' - parts = [ - "Application Layer Protocol: %s" % alp, + "Application Layer Protocol: %s" % self.alp, "Cipher: %s, %s bit, %s" % self.cipher, "SSL certificate chain:" ] @@ -155,13 +150,14 @@ class Pathoc(tcp.TCPClient): # SSL ssl=None, sni=None, - sslversion=4, + sslversion='SSLv23', clientcert=None, ciphers=None, # HTTP/2 use_http2=False, http2_skip_connection_preface=False, + http2_framedump = False, # Websockets ws_read_limit = None, @@ -199,6 +195,7 @@ class Pathoc(tcp.TCPClient): self.use_http2 = use_http2 self.http2_skip_connection_preface = http2_skip_connection_preface + self.http2_framedump = http2_framedump self.ws_read_limit = ws_read_limit @@ -216,6 +213,9 @@ class Pathoc(tcp.TCPClient): self.ws_framereader = None if self.use_http2: + if not OpenSSL._util.lib.Cryptography_HAS_ALPN: # pragma: nocover + print >> sys.stderr, "HTTP/2 requires ALPN support. Please use OpenSSL >= 1.0.2." + print >> sys.stderr, "Pathoc might not be working as expected without ALPN." self.protocol = http2.HTTP2Protocol(self) else: # TODO: create HTTP or Websockets protocol @@ -259,7 +259,7 @@ class Pathoc(tcp.TCPClient): an HTTP CONNECT request. """ if self.use_http2 and not self.ssl: - raise ValueError("HTTP2 without SSL is not supported.") + raise NotImplementedError("HTTP2 without SSL is not supported.") tcp.TCPClient.connect(self) @@ -294,7 +294,7 @@ class Pathoc(tcp.TCPClient): if self.use_http2: self.protocol.check_alpn() if not self.http2_skip_connection_preface: - self.protocol.perform_connection_preface() + self.protocol.perform_client_connection_preface() if self.timeout: self.settimeout(self.timeout) @@ -462,6 +462,7 @@ def main(args): # pragma: nocover ciphers = args.ciphers, use_http2 = args.use_http2, http2_skip_connection_preface = args.http2_skip_connection_preface, + http2_framedump = args.http2_framedump, showreq = args.showreq, showresp = args.showresp, explain = args.explain, diff --git a/libpathod/pathoc_cmdline.py b/libpathod/pathoc_cmdline.py new file mode 100644 index 000000000..1d0df3b55 --- /dev/null +++ b/libpathod/pathoc_cmdline.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python +import sys +import argparse +import os +import os.path +import re +from netlib import http_uastrings +from . import pathoc, pathod, version, utils, language + + +def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): + preparser = argparse.ArgumentParser(add_help=False) + preparser.add_argument( + "--show-uas", dest="showua", action="store_true", default=False, + help="Print user agent shortcuts and exit." + ) + pa = preparser.parse_known_args(argv)[0] + if pa.showua: + print >> stdout, "User agent strings:" + for i in http_uastrings.UASTRINGS: + print >> stdout, " ", i[1], i[0] + sys.exit(0) + + parser = argparse.ArgumentParser( + description='A perverse HTTP client.', parents=[preparser] + ) + parser.add_argument( + '--version', + action='version', + version="pathoc " + version.VERSION + ) + parser.add_argument( + "-c", dest="connect_to", type=str, default=False, + metavar = "HOST:PORT", + help="Issue an HTTP CONNECT to connect to the specified host." + ) + parser.add_argument( + "--memo-limit", dest='memolimit', default=5000, type=int, metavar="N", + help='Stop if we do not find a valid request after N attempts.' + ) + parser.add_argument( + "-m", dest='memo', action="store_true", default=False, + help=""" + Remember specs, and never play the same one twice. Note that this + means requests have to be rendered in memory, which means that + large generated data can cause issues. + """ + ) + parser.add_argument( + "-n", dest='repeat', default=1, type=int, metavar="N", + help='Repeat N times. If 0 repeat for ever.' + ) + parser.add_argument( + "-w", dest='wait', default=0, type=float, metavar="N", + help='Wait N seconds between each request.' + ) + parser.add_argument( + "-r", dest="random", action="store_true", default=False, + help=""" + Select a random request from those specified. If this is not specified, + requests are all played in sequence. + """ + ) + parser.add_argument( + "-t", dest="timeout", type=int, default=None, + help="Connection timeout" + ) + parser.add_argument( + "--http2", dest="use_http2", action="store_true", default=False, + help='Perform all requests over a single HTTP/2 connection.' + ) + parser.add_argument( + "--http2-skip-connection-preface", + dest="http2_skip_connection_preface", + action="store_true", + default=False, + help='Skips the HTTP/2 connection preface before sending requests.') + + parser.add_argument( + 'host', type=str, + metavar = "host[:port]", + help='Host and port to connect to' + ) + parser.add_argument( + 'requests', type=str, nargs="+", + help=""" + Request specification, or path to a file containing request + specifcations + """ + ) + + group = parser.add_argument_group( + 'SSL', + ) + group.add_argument( + "-s", dest="ssl", action="store_true", default=False, + help="Connect with SSL" + ) + group.add_argument( + "-C", dest="clientcert", type=str, default=False, + help="Path to a file containing client certificate and private key" + ) + group.add_argument( + "-i", dest="sni", type=str, default=False, + help="SSL Server Name Indication" + ) + group.add_argument( + "--ciphers", dest="ciphers", type=str, default=False, + help="SSL cipher specification" + ) + group.add_argument( + "--sslversion", dest="sslversion", type=str, default='SSLv23', + choices=utils.SSLVERSIONS.keys(), + help="""" + Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. + Default to SSLv23.""" + ) + + group = parser.add_argument_group( + 'Controlling Output', + """ + Some of these options expand generated values for logging - if + you're generating large data, use them with caution. + """ + ) + group.add_argument( + "-I", dest="ignorecodes", type=str, default="", + help="Comma-separated list of response codes to ignore" + ) + group.add_argument( + "-S", dest="showssl", action="store_true", default=False, + help="Show info on SSL connection" + ) + group.add_argument( + "-e", dest="explain", action="store_true", default=False, + help="Explain requests" + ) + group.add_argument( + "-o", dest="oneshot", action="store_true", default=False, + help="Oneshot - exit after first non-ignored response" + ) + group.add_argument( + "-q", dest="showreq", action="store_true", default=False, + help="Print full request" + ) + group.add_argument( + "-p", dest="showresp", action="store_true", default=False, + help="Print full response" + ) + group.add_argument( + "-T", dest="ignoretimeout", action="store_true", default=False, + help="Ignore timeouts" + ) + group.add_argument( + "-x", dest="hexdump", action="store_true", default=False, + help="Output in hexdump format" + ) + group.add_argument( + "--http2-framedump", dest="http2_framedump", action="store_true", default=False, + help="Output all received & sent HTTP/2 frames" + ) + + args = parser.parse_args(argv[1:]) + + args.port = None + if ":" in args.host: + h, p = args.host.rsplit(":", 1) + try: + p = int(p) + except ValueError: + return parser.error("Invalid port in host spec: %s" % args.host) + args.host = h + args.port = p + + if args.port is None: + args.port = 443 if args.ssl else 80 + + try: + args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i] + except ValueError: + return parser.error( + "Invalid return code specification: %s" % + args.ignorecodes) + + if args.connect_to: + parts = args.connect_to.split(":") + if len(parts) != 2: + return parser.error( + "Invalid CONNECT specification: %s" % + args.connect_to) + try: + parts[1] = int(parts[1]) + except ValueError: + return parser.error( + "Invalid CONNECT specification: %s" % + args.connect_to) + args.connect_to = parts + else: + args.connect_to = None + + if args.http2_skip_connection_preface: + args.use_http2 = True + + if args.use_http2: + args.ssl = True + + reqs = [] + for r in args.requests: + if os.path.isfile(r): + data = open(r).read() + r = data + try: + reqs.append(language.parse_pathoc(r, args.use_http2)) + except language.ParseException as v: + print >> stderr, "Error parsing request spec: %s" % v.msg + print >> stderr, v.marked() + sys.exit(1) + args.requests = reqs + return args + + +def go_pathoc(): # pragma: nocover + args = args_pathoc(sys.argv) + pathoc.main(args) diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 13f602b4d..212abbdcc 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -7,7 +7,7 @@ import urllib import re import time -from netlib import tcp, http, wsgi, certutils, websockets +from netlib import tcp, http, http2, wsgi, certutils, websockets, odict from . import version, app, language, utils, log import language.http @@ -20,7 +20,7 @@ DEFAULT_CERT_DOMAIN = "pathod.net" CONFDIR = "~/.mitmproxy" CERTSTORE_BASENAME = "mitmproxy" CA_CERT_NAME = "mitmproxy-ca.pem" -DEFAULT_ANCHOR = r"/p/?" +DEFAULT_CRAFT_ANCHOR = "/p/" logger = logging.getLogger('pathod') @@ -39,21 +39,23 @@ class SSLOptions: request_client_cert=False, sslversion=tcp.SSLv23_METHOD, ciphers=None, - certs=None + certs=None, + alpn_select=http2.HTTP2Protocol.ALPN_PROTO_H2, ): self.confdir = confdir self.cn = cn + self.sans = sans + self.not_after_connect = not_after_connect + self.request_client_cert = request_client_cert + self.sslversion = sslversion + self.ciphers = ciphers + self.alpn_select = alpn_select self.certstore = certutils.CertStore.from_store( os.path.expanduser(confdir), CERTSTORE_BASENAME ) for i in certs or []: self.certstore.add_cert_file(*i) - self.not_after_connect = not_after_connect - self.request_client_cert = request_client_cert - self.ciphers = ciphers - self.sslversion = sslversion - self.sans = sans def get_cert(self, name): if self.cn: @@ -67,32 +69,37 @@ class PathodHandler(tcp.BaseHandler): wbufsize = 0 sni = None - def __init__(self, connection, address, server, logfp, settings): - self.logfp = logfp + def __init__(self, connection, address, server, logfp, settings, http2_framedump=False): tcp.BaseHandler.__init__(self, connection, address, server) + self.logfp = logfp self.settings = copy.copy(settings) + self.protocol = None + self.use_http2 = False + self.http2_framedump = http2_framedump - def handle_sni(self, connection): + def _handle_sni(self, connection): self.sni = connection.get_servername() def http_serve_crafted(self, crafted): + """ + This method is HTTP/1 and HTTP/2 capable. + """ + error, crafted = self.server.check_policy( crafted, self.settings ) if error: - err = language.http.make_error_response(error) + err = self.make_http_error_response(error) language.serve(err, self.wfile, self.settings) return None, dict( type="error", msg = error ) - if self.server.explain and not isinstance( - crafted, - language.http.PathodErrorResponse - ): + if self.server.explain and not hasattr(crafted, 'is_error_response'): crafted = crafted.freeze(self.settings) log.write(self.logfp, ">> Spec: %s" % crafted.spec()) + response_log = language.serve( crafted, self.wfile, @@ -152,6 +159,8 @@ class PathodHandler(tcp.BaseHandler): def handle_http_connect(self, connect, lg): """ + This method is HTTP/1 only. + Handle a CONNECT request. """ http.read_headers(self.rfile) @@ -169,10 +178,11 @@ class PathodHandler(tcp.BaseHandler): self.convert_to_ssl( cert, key, - handle_sni=self.handle_sni, + handle_sni=self._handle_sni, request_client_cert=self.server.ssloptions.request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.sslversion, + alpn_select=self.server.ssloptions.alpn_select, ) except tcp.NetLibError as v: s = str(v) @@ -182,10 +192,12 @@ class PathodHandler(tcp.BaseHandler): def handle_http_app(self, method, path, headers, content, lg): """ + This method is HTTP/1 only. + Handle a request to the built-in app. """ if self.server.noweb: - crafted = language.http.make_error_response("Access Denied") + crafted = self.make_http_error_response("Access Denied") language.serve(crafted, self.wfile, self.settings) return None, dict( type="error", @@ -206,6 +218,8 @@ class PathodHandler(tcp.BaseHandler): def handle_http_request(self): """ + This method is HTTP/1 and HTTP/2 capable. + Returns a (handler, log) tuple. handler: Handler for the next request, or None to disconnect @@ -214,28 +228,26 @@ class PathodHandler(tcp.BaseHandler): lr = self.rfile if self.server.logreq else None lw = self.wfile if self.server.logresp else None with log.Log(self.logfp, self.server.hexdump, lr, lw) as lg: - line = http.get_request_line(self.rfile) - if not line: - # Normal termination - return None, None - - m = utils.MemBool() - if m(http.parse_init_connect(line)): - return self.handle_http_connect(m.v, lg) - elif m(http.parse_init_proxy(line)): - method, _, _, _, path, httpversion = m.v - elif m(http.parse_init_http(line)): - method, path, httpversion = m.v + if self.use_http2: + self.protocol.perform_server_connection_preface() + stream_id, headers, body = self.protocol.read_request() + method = headers[':method'] + path = headers[':path'] + headers = odict.ODict(headers) + httpversion = "" else: - s = "Invalid first line: %s" % repr(line) - lg(s) - return None, dict(type="error", msg=s) - - headers = http.read_headers(self.rfile) - if headers is None: - s = "Invalid headers" - lg(s) - return None, dict(type="error", msg=s) + req = self.read_http_request(lg) + if 'next_handle' in req: + return req['next_handle'] + if 'errors' in req: + return None, req['errors'] + if not 'method' in req or not 'path' in req: + return None, None + method = req['method'] + path = req['path'] + headers = req['headers'] + body = req['body'] + httpversion = req['httpversion'] clientcert = None if self.clientcert: @@ -265,16 +277,6 @@ class PathodHandler(tcp.BaseHandler): if self.ssl_established: retlog["cipher"] = self.get_current_cipher() - try: - content = http.read_http_body( - self.rfile, headers, None, - method, None, True - ) - except http.HttpError as s: - s = str(s) - lg(s) - return None, dict(type="error", msg=s) - m = utils.MemBool() websocket_key = websockets.check_client_handshake(headers) self.settings.websocket_key = websocket_key @@ -285,27 +287,40 @@ class PathodHandler(tcp.BaseHandler): anchor_gen = language.parse_pathod("ws") else: anchor_gen = None - for i in self.server.anchors: - if i[0].match(path): - anchor_gen = i[1] + + for regex, spec in self.server.anchors: + if regex.match(path): + anchor_gen = language.parse_pathod(spec, self.use_http2) break else: - if m(self.server.craftanchor.match(path)): - spec = urllib.unquote(path)[len(m.v.group()):] + if m(path.startswith(self.server.craftanchor)): + spec = urllib.unquote(path)[len(self.server.craftanchor):] if spec: try: - anchor_gen = language.parse_pathod(spec) + anchor_gen = language.parse_pathod(spec, self.use_http2) except language.ParseException as v: lg("Parse error: %s" % v.msg) - anchor_gen = iter([language.http.make_error_response( + anchor_gen = iter([self.make_http_error_response( "Parse Error", "Error parsing response spec: %s\n" % ( v.msg + v.marked() ) )]) + else: + if self.use_http2: + anchor_gen = iter([self.make_http_error_response( + "Spec Error", + "HTTP/2 only supports request/response with the craft anchor point: %s" % + self.server.craftanchor + )]) + if anchor_gen: spec = anchor_gen.next() + + if self.use_http2 and isinstance(spec, language.http2.Response): + spec.stream_id = stream_id + lg("crafting spec: %s" % spec) nexthandler, retlog["response"] = self.http_serve_crafted( spec @@ -315,7 +330,113 @@ class PathodHandler(tcp.BaseHandler): else: return nexthandler, retlog else: - return self.handle_http_app(method, path, headers, content, lg) + return self.handle_http_app(method, path, headers, body, lg) + + def read_http_request(self, lg): + """ + This method is HTTP/1 only. + """ + line = http.get_request_line(self.rfile) + if not line: + # Normal termination + return dict() + + m = utils.MemBool() + if m(http.parse_init_connect(line)): + return dict(next_handle=self.handle_http_connect(m.v, lg)) + elif m(http.parse_init_proxy(line)): + method, _, _, _, path, httpversion = m.v + elif m(http.parse_init_http(line)): + method, path, httpversion = m.v + else: + s = "Invalid first line: %s" % repr(line) + lg(s) + return dict(errors=dict(type="error", msg=s)) + + headers = http.read_headers(self.rfile) + if headers is None: + s = "Invalid headers" + lg(s) + return dict(errors=dict(type="error", msg=s)) + + try: + body = http.read_http_body( + self.rfile, + headers, + None, + method, + None, + True, + ) + except http.HttpError as s: + s = str(s) + lg(s) + return dict(errors=dict(type="error", msg=s)) + + return dict( + method=method, + path=path, + headers=headers, + body=body, + httpversion=httpversion) + + def make_http_error_response(self, reason, body=None): + """ + This method is HTTP/1 and HTTP/2 capable. + """ + if self.use_http2: + resp = language.http2.make_error_response(reason, body) + else: + resp = language.http.make_error_response(reason, body) + resp.is_error_response = True + return resp + + def handle(self): + self.settimeout(self.server.timeout) + + if self.server.ssl: + try: + cert, key, _ = self.server.ssloptions.get_cert(None) + self.convert_to_ssl( + cert, + key, + dhparams=self.server.ssloptions.certstore.dhparams, + handle_sni=self._handle_sni, + request_client_cert=self.server.ssloptions.request_client_cert, + cipher_list=self.server.ssloptions.ciphers, + method=self.server.ssloptions.sslversion, + alpn_select=self.server.ssloptions.alpn_select, + ) + except tcp.NetLibError as v: + s = str(v) + self.server.add_log( + dict( + type="error", + msg=s + ) + ) + log.write(self.logfp, s) + return + + alp = self.get_alpn_proto_negotiated() + if alp == http2.HTTP2Protocol.ALPN_PROTO_H2: + self.protocol = http2.HTTP2Protocol(self, is_server=True, dump_frames=self.http2_framedump) + self.use_http2 = True + + # if not self.protocol: + # # TODO: create HTTP or Websockets protocol + # self.protocol = None + + self.settings.protocol = self.protocol + + handler = self.handle_http_request + + while not self.finished: + handler, l = handler() + if l: + self.addlog(l) + if not handler: + return def addlog(self, log): # FIXME: The bytes in the log should not be escaped. We do this at the @@ -329,37 +450,6 @@ class PathodHandler(tcp.BaseHandler): log["response_bytes"] = bytes self.server.add_log(log) - def handle(self): - if self.server.ssl: - try: - cert, key, _ = self.server.ssloptions.get_cert(None) - self.convert_to_ssl( - cert, - key, - handle_sni=self.handle_sni, - request_client_cert=self.server.ssloptions.request_client_cert, - cipher_list=self.server.ssloptions.ciphers, - method=self.server.ssloptions.sslversion, - ) - except tcp.NetLibError as v: - s = str(v) - self.server.add_log( - dict( - type="error", - msg=s - ) - ) - log.write(self.logfp, s) - return - self.settimeout(self.server.timeout) - handler = self.handle_http_request - while not self.finished: - handler, l = handler() - if l: - self.addlog(l) - if not handler: - return - class Pathod(tcp.TCPServer): LOGBUF = 500 @@ -369,7 +459,7 @@ class Pathod(tcp.TCPServer): addr, ssl=False, ssloptions=None, - craftanchor=re.compile(DEFAULT_ANCHOR), + craftanchor=DEFAULT_CRAFT_ANCHOR, staticdir=None, anchors=(), sizelimit=None, @@ -382,6 +472,7 @@ class Pathod(tcp.TCPServer): logresp=False, explain=False, hexdump=False, + http2_framedump=False, webdebug=False, logfp=sys.stdout, ): @@ -389,7 +480,7 @@ class Pathod(tcp.TCPServer): addr: (address, port) tuple. If port is 0, a free port will be automatically chosen. ssloptions: an SSLOptions object. - craftanchor: string specifying the path under which to anchor + craftanchor: URL prefix specifying the path under which to anchor response generation. staticdir: path to a directory of static resources, or None. anchors: List of (regex object, language.Request object) tuples, or @@ -409,6 +500,7 @@ class Pathod(tcp.TCPServer): self.noapi, self.nohang = noapi, nohang self.timeout, self.logreq = timeout, logreq self.logresp, self.hexdump = logresp, hexdump + self.http2_framedump = http2_framedump self.explain = explain self.logfp = logfp @@ -446,7 +538,8 @@ class Pathod(tcp.TCPServer): client_address, self, self.logfp, - self.settings + self.settings, + self.http2_framedump, ) try: h.handle() @@ -502,7 +595,7 @@ def main(args): # pragma: nocover ciphers = args.ciphers, sslversion = utils.SSLVERSIONS[args.sslversion], certs = args.ssl_certs, - sans = args.sans + sans = args.sans, ) root = logging.getLogger() @@ -542,6 +635,7 @@ def main(args): # pragma: nocover logreq = args.logreq, logresp = args.logresp, hexdump = args.hexdump, + http2_framedump = args.http2_framedump, explain = args.explain, webdebug = args.webdebug ) diff --git a/libpathod/pathod_cmdline.py b/libpathod/pathod_cmdline.py new file mode 100644 index 000000000..4343401ff --- /dev/null +++ b/libpathod/pathod_cmdline.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +import sys +import argparse +import os +import os.path +import re +from netlib import http_uastrings +from . import pathoc, pathod, version, utils, language + + +def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr): + parser = argparse.ArgumentParser( + description='A pathological HTTP/S daemon.' + ) + parser.add_argument( + '--version', + action='version', + version="pathod " + version.VERSION + ) + parser.add_argument( + "-p", + dest='port', + default=9999, + type=int, + help='Port. Specify 0 to pick an arbitrary empty port. (9999)' + ) + parser.add_argument( + "-l", + dest='address', + default="127.0.0.1", + type=str, + help='Listening address. (127.0.0.1)' + ) + parser.add_argument( + "-a", + dest='anchors', + default=[], + type=str, + action="append", + metavar="ANCHOR", + help=""" + Add an anchor. Specified as a string with the form + pattern=spec or pattern=filepath, where pattern is a regular + expression. + """ + ) + parser.add_argument( + "-c", dest='craftanchor', default=pathod.DEFAULT_CRAFT_ANCHOR, type=str, + help=""" + URL path specifying prefix for URL crafting + commands. (%s) + """%pathod.DEFAULT_CRAFT_ANCHOR + ) + parser.add_argument( + "--confdir", + action="store", type = str, dest="confdir", default='~/.mitmproxy', + help = "Configuration directory. (~/.mitmproxy)" + ) + parser.add_argument( + "-d", dest='staticdir', default=None, type=str, + help='Directory for static files.' + ) + parser.add_argument( + "-D", dest='daemonize', default=False, action="store_true", + help='Daemonize.' + ) + parser.add_argument( + "-t", dest="timeout", type=int, default=None, + help="Connection timeout" + ) + parser.add_argument( + "--limit-size", + dest='sizelimit', + default=None, + type=str, + help='Size limit of served responses. Understands size suffixes, i.e. 100k.') + parser.add_argument( + "--noapi", dest='noapi', default=False, action="store_true", + help='Disable API.' + ) + parser.add_argument( + "--nohang", dest='nohang', default=False, action="store_true", + help='Disable pauses during crafted response generation.' + ) + parser.add_argument( + "--noweb", dest='noweb', default=False, action="store_true", + help='Disable both web interface and API.' + ) + parser.add_argument( + "--nocraft", + dest='nocraft', + default=False, + action="store_true", + help='Disable response crafting. If anchors are specified, they still work.') + parser.add_argument( + "--webdebug", dest='webdebug', default=False, action="store_true", + help='Debugging mode for the web app (dev only).' + ) + + group = parser.add_argument_group( + 'SSL', + ) + group.add_argument( + "-s", dest='ssl', default=False, action="store_true", + help='Run in HTTPS mode.' + ) + group.add_argument( + "--cn", + dest="cn", + type=str, + default=None, + help="CN for generated SSL certs. Default: %s" % + pathod.DEFAULT_CERT_DOMAIN) + group.add_argument( + "-C", dest='ssl_not_after_connect', default=False, action="store_true", + help="Don't expect SSL after a CONNECT request." + ) + group.add_argument( + "--cert", dest='ssl_certs', default=[], type=str, + metavar = "SPEC", action="append", + help = """ + Add an SSL certificate. SPEC is of the form "[domain=]path". The domain + may include a wildcard, and is equal to "*" if not specified. The file + at path is a certificate in PEM format. If a private key is included in + the PEM, it is used, else the default key in the conf dir is used. Can + be passed multiple times. + """ + ) + group.add_argument( + "--ciphers", dest="ciphers", type=str, default=False, + help="SSL cipher specification" + ) + group.add_argument( + "--san", dest="sans", type=str, default=[], action="append", + metavar="SAN", + help=""" + Subject Altnernate Name to add to the server certificate. + May be passed multiple times. + """ + ) + group.add_argument( + "--sslversion", dest="sslversion", type=str, default='SSLv23', + choices=utils.SSLVERSIONS.keys(), + help="""" + Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. + Default to SSLv23.""" + ) + + group = parser.add_argument_group( + 'Controlling Logging', + """ + Some of these options expand generated values for logging - if + you're generating large data, use them with caution. + """ + ) + group.add_argument( + "-e", dest="explain", action="store_true", default=False, + help="Explain responses" + ) + group.add_argument( + "-f", dest='logfile', default=None, type=str, + help='Log to file.' + ) + group.add_argument( + "-q", dest="logreq", action="store_true", default=False, + help="Log full request" + ) + group.add_argument( + "-r", dest="logresp", action="store_true", default=False, + help="Log full response" + ) + group.add_argument( + "-x", dest="hexdump", action="store_true", default=False, + help="Log request/response in hexdump format" + ) + group.add_argument( + "--http2-framedump", dest="http2_framedump", action="store_true", default=False, + help="Output all received & sent HTTP/2 frames" + ) + + + args = parser.parse_args(argv[1:]) + + certs = [] + for i in args.ssl_certs: + parts = i.split("=", 1) + if len(parts) == 1: + parts = ["*", parts[0]] + parts[1] = os.path.expanduser(parts[1]) + if not os.path.isfile(parts[1]): + return parser.error( + "Certificate file does not exist: %s" % + parts[1]) + certs.append(parts) + args.ssl_certs = certs + + alst = [] + for i in args.anchors: + parts = utils.parse_anchor_spec(i) + if not parts: + return parser.error("Invalid anchor specification: %s" % i) + alst.append(parts) + args.anchors = alst + + sizelimit = None + if args.sizelimit: + try: + sizelimit = utils.parse_size(args.sizelimit) + except ValueError as v: + return parser.error(v) + args.sizelimit = sizelimit + + anchors = [] + for patt, spec in args.anchors: + if os.path.isfile(spec): + data = open(spec).read() + spec = data + try: + arex = re.compile(patt) + except re.error: + return parser.error("Invalid regex in anchor: %s" % patt) + anchors.append((arex, spec)) + args.anchors = anchors + return args + + +def go_pathod(): # pragma: nocover + args = args_pathod(sys.argv) + pathod.main(args) diff --git a/libpathod/utils.py b/libpathod/utils.py index 9bd2812e5..481c51372 100644 --- a/libpathod/utils.py +++ b/libpathod/utils.py @@ -3,10 +3,12 @@ import sys from netlib import tcp SSLVERSIONS = { - 1: tcp.TLSv1_METHOD, - 2: tcp.SSLv2_METHOD, - 3: tcp.SSLv3_METHOD, - 4: tcp.SSLv23_METHOD, + 'TLSv1.2': tcp.TLSv1_2_METHOD, + 'TLSv1.1': tcp.TLSv1_1_METHOD, + 'TLSv1': tcp.TLSv1_METHOD, + 'SSLv3': tcp.SSLv3_METHOD, + 'SSLv2': tcp.SSLv2_METHOD, + 'SSLv23': tcp.SSLv23_METHOD, } SIZE_UNITS = dict( diff --git a/pathoc b/pathoc index cbf8f7738..b31216113 100755 --- a/pathoc +++ b/pathoc @@ -1,5 +1,6 @@ #!/usr/bin/env python -from libpathod import cmdline + +from libpathod import pathoc_cmdline as cmdline if __name__ == "__main__": cmdline.go_pathoc() diff --git a/pathod b/pathod index ca0baa573..a79becf1f 100755 --- a/pathod +++ b/pathod @@ -1,5 +1,6 @@ #!/usr/bin/env python -from libpathod import cmdline + +from libpathod import pathod_cmdline as cmdline if __name__ == "__main__": cmdline.go_pathod() diff --git a/setup.py b/setup.py index 344712eb3..041b164da 100644 --- a/setup.py +++ b/setup.py @@ -40,8 +40,8 @@ setup( include_package_data=True, entry_points={ 'console_scripts': [ - "pathod = libpathod.cmdline:go_pathod", - "pathoc = libpathod.cmdline:go_pathoc" + "pathod = libpathod.pathod_cmdline:go_pathod", + "pathoc = libpathod.pathoc_cmdline:go_pathoc" ] }, install_requires=[ diff --git a/test/test_cmdline.py b/test/test_cmdline.py deleted file mode 100644 index c1b556082..000000000 --- a/test/test_cmdline.py +++ /dev/null @@ -1,155 +0,0 @@ -from libpathod import cmdline -import tutils -import cStringIO -import mock - - -@mock.patch("argparse.ArgumentParser.error") -def test_pathod(perror): - assert cmdline.args_pathod(["pathod"]) - - a = cmdline.args_pathod( - [ - "pathod", - "--cert", - tutils.test_data.path("data/testkey.pem") - ] - ) - assert a.ssl_certs - - a = cmdline.args_pathod( - [ - "pathod", - "--cert", - "nonexistent" - ] - ) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathod( - [ - "pathod", - "-a", - "foo=200" - ] - ) - assert a.anchors - - a = cmdline.args_pathod( - [ - "pathod", - "-a", - "foo=" + tutils.test_data.path("data/response") - ] - ) - assert a.anchors - - a = cmdline.args_pathod( - [ - "pathod", - "-a", - "?=200" - ] - ) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathod( - [ - "pathod", - "-a", - "foo" - ] - ) - assert perror.called - perror.reset_mock() - - s = cStringIO.StringIO() - tutils.raises( - SystemExit, - cmdline.args_pathod, - ["pathod", "-a", "foo=."], - s, - s - ) - - a = cmdline.args_pathod( - [ - "pathod", - "--limit-size", - "200k" - ] - ) - assert a.sizelimit - - a = cmdline.args_pathod( - [ - "pathod", - "--limit-size", - "q" - ] - ) - assert perror.called - perror.reset_mock() - - -@mock.patch("argparse.ArgumentParser.error") -def test_pathoc(perror): - assert cmdline.args_pathoc(["pathoc", "foo.com", "get:/"]) - s = cStringIO.StringIO() - tutils.raises( - SystemExit, cmdline.args_pathoc, [ - "pathoc", "--show-uas"], s, s) - - a = cmdline.args_pathoc(["pathoc", "foo.com:8888", "get:/"]) - assert a.port == 8888 - - a = cmdline.args_pathoc(["pathoc", "foo.com:xxx", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc(["pathoc", "-I", "10, 20", "foo.com:8888", "get:/"]) - assert a.ignorecodes == [10, 20] - - a = cmdline.args_pathoc(["pathoc", "-I", "xx, 20", "foo.com:8888", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc(["pathoc", "-c", "foo:10", "foo.com:8888", "get:/"]) - assert a.connect_to == ["foo", 10] - - a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc( - ["pathoc", "-c", "foo:bar", "foo.com:8888", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc( - [ - "pathoc", - "foo.com:8888", - tutils.test_data.path("data/request") - ] - ) - assert len(list(a.requests)) == 1 - - a = cmdline.args_pathod( - [ - "pathod", - "-c", - "?" - ] - ) - assert perror.called - perror.reset_mock() - - tutils.raises( - SystemExit, - cmdline.args_pathoc, - ["pathoc", "foo.com", "invalid"], - s, s - ) diff --git a/test/test_language_http2.py b/test/test_language_http2.py new file mode 100644 index 000000000..0be42253c --- /dev/null +++ b/test/test_language_http2.py @@ -0,0 +1,177 @@ +import cStringIO + +from netlib import tcp +from libpathod import language +from libpathod.language import http2, base +import netlib +import tutils + + +def parse_request(s): + return language.parse_pathoc(s, True).next() + +def parse_response(s): + return language.parse_pathod(s, True).next() + +def default_settings(): + return language.Settings( + request_host = "foo.com", + protocol = netlib.http2.HTTP2Protocol(tcp.TCPClient(('localhost', 1234))) + ) + + +def test_make_error_response(): + d = cStringIO.StringIO() + s = http2.make_error_response("foo", "bar") + language.serve(s, d, default_settings()) + + +class TestRequest: + def test_cached_values(self): + req = parse_request("get:/") + req_id = id(req) + assert req_id == id(req.resolve(default_settings())) + assert req.values(default_settings()) == req.values(default_settings()) + + def test_nonascii(self): + tutils.raises("ascii", parse_request, "get:\xf0") + + def test_err(self): + tutils.raises(language.ParseException, parse_request, 'GET') + + def test_simple(self): + r = parse_request('GET:"/foo"') + assert r.method.string() == "GET" + assert r.path.string() == "/foo" + r = parse_request('GET:/foo') + assert r.path.string() == "/foo" + + def test_multiple(self): + r = list(language.parse_pathoc("GET:/ PUT:/")) + assert r[0].method.string() == "GET" + assert r[1].method.string() == "PUT" + assert len(r) == 2 + + l = """ + GET + "/foo" + + PUT + + "/foo + + + + bar" + """ + r = list(language.parse_pathoc(l, True)) + assert len(r) == 2 + assert r[0].method.string() == "GET" + assert r[1].method.string() == "PUT" + + l = """ + get:"http://localhost:9999/p/200" + get:"http://localhost:9999/p/200" + """ + r = list(language.parse_pathoc(l, True)) + assert len(r) == 2 + assert r[0].method.string() == "GET" + assert r[1].method.string() == "GET" + + def test_render_simple(self): + s = cStringIO.StringIO() + r = parse_request("GET:'/foo'") + assert language.serve( + r, + s, + default_settings(), + ) + + def test_render_with_headers(self): + s = cStringIO.StringIO() + r = parse_request('GET:/foo:h"foo"="bar"') + assert language.serve( + r, + s, + default_settings(), + ) + + def test_render_with_body(self): + s = cStringIO.StringIO() + r = parse_request("GET:'/foo':bfoobar") + assert language.serve( + r, + s, + default_settings(), + ) + + def test_spec(self): + def rt(s): + s = parse_request(s).spec() + assert parse_request(s).spec() == s + rt("get:/foo") + + +class TestResponse: + def test_cached_values(self): + res = parse_response("200") + res_id = id(res) + assert res_id == id(res.resolve(default_settings())) + assert res.values(default_settings()) == res.values(default_settings()) + + def test_nonascii(self): + tutils.raises("ascii", parse_response, "200:\xf0") + + def test_err(self): + tutils.raises(language.ParseException, parse_response, 'GET:/') + + def test_simple(self): + r = parse_response('200') + assert r.code.string() == "200" + assert len(r.headers) == 0 + + r = parse_response('200:h"foo"="bar"') + assert r.code.string() == "200" + assert len(r.headers) == 1 + assert r.headers[0].values(default_settings()) == ("foo", "bar") + assert r.body == None + + r = parse_response('200:h"foo"="bar":bfoobar:h"bla"="fasel"') + assert r.code.string() == "200" + assert len(r.headers) == 2 + assert r.headers[0].values(default_settings()) == ("foo", "bar") + assert r.headers[1].values(default_settings()) == ("bla", "fasel") + assert r.body.string() == "foobar" + + def test_render_simple(self): + s = cStringIO.StringIO() + r = parse_response('200') + assert language.serve( + r, + s, + default_settings(), + ) + + def test_render_with_headers(self): + s = cStringIO.StringIO() + r = parse_response('200:h"foo"="bar"') + assert language.serve( + r, + s, + default_settings(), + ) + + def test_render_with_body(self): + s = cStringIO.StringIO() + r = parse_response('200:bfoobar') + assert language.serve( + r, + s, + default_settings(), + ) + + def test_spec(self): + def rt(s): + s = parse_response(s).spec() + assert parse_response(s).spec() == s + rt("200:bfoobar") diff --git a/test/test_pathoc.py b/test/test_pathoc.py index c00c7d535..d39f92759 100644 --- a/test/test_pathoc.py +++ b/test/test_pathoc.py @@ -1,8 +1,9 @@ import json import cStringIO import re +from mock import Mock -from netlib import tcp, http +from netlib import tcp, http, http2 from libpathod import pathoc, test, version, pathod, language import tutils @@ -22,7 +23,7 @@ class _TestDaemon: ssloptions = self.ssloptions, staticdir = tutils.test_data.path("data"), anchors = [ - (re.compile("/anchor/.*"), language.parse_pathod("202")) + (re.compile("/anchor/.*"), "202") ] ) @@ -86,8 +87,9 @@ class _TestDaemon: class TestDaemonSSL(_TestDaemon): ssl = True ssloptions = pathod.SSLOptions( - request_client_cert=True, - sans = ["test1.com", "test2.com"] + request_client_cert = True, + sans = ["test1.com", "test2.com"], + alpn_select = http2.HTTP2Protocol.ALPN_PROTO_H2, ) def test_sni(self): @@ -119,6 +121,14 @@ class TestDaemonSSL(_TestDaemon): d = json.loads(r.content) assert d["log"][0]["request"]["clientcert"]["keyinfo"] + def test_http2_without_ssl(self): + c = pathoc.Pathoc( + ("127.0.0.1", self.d.port), + use_http2 = True, + ssl = False, + ) + tutils.raises(NotImplementedError, c.connect) + class TestDaemon(_TestDaemon): ssl = False @@ -216,3 +226,46 @@ class TestDaemon(_TestDaemon): "HTTP/1.1 200 OK\r\n" ) c.http_connect(to) + + +class TestDaemonHTTP2(_TestDaemon): + ssl = True + + def test_http2(self): + c = pathoc.Pathoc( + ("127.0.0.1", self.d.port), + use_http2 = True, + ssl = True, + ) + assert isinstance(c.protocol, http2.HTTP2Protocol) + + c = pathoc.Pathoc( + ("127.0.0.1", self.d.port), + ) + assert c.protocol == None # TODO: change if other protocols get implemented + + def test_http2_alpn(self): + c = pathoc.Pathoc( + ("127.0.0.1", self.d.port), + ssl = True, + use_http2 = True, + http2_skip_connection_preface = True, + ) + + tmp_convert_to_ssl = c.convert_to_ssl + c.convert_to_ssl = Mock() + c.convert_to_ssl.side_effect = tmp_convert_to_ssl + c.connect() + + _, kwargs = c.convert_to_ssl.call_args + assert set(kwargs['alpn_protos']) == set([b'http1.1', b'h2']) + + def test_request(self): + c = pathoc.Pathoc( + ("127.0.0.1", self.d.port), + ssl = True, + use_http2 = True, + ) + c.connect() + resp = c.request("get:/p/200") + assert resp.status_code == "200" diff --git a/test/test_pathoc_cmdline.py b/test/test_pathoc_cmdline.py new file mode 100644 index 000000000..6c070aed9 --- /dev/null +++ b/test/test_pathoc_cmdline.py @@ -0,0 +1,64 @@ +from libpathod import pathoc_cmdline as cmdline +import tutils +import cStringIO +import mock + + +@mock.patch("argparse.ArgumentParser.error") +def test_pathoc(perror): + assert cmdline.args_pathoc(["pathoc", "foo.com", "get:/"]) + s = cStringIO.StringIO() + tutils.raises( + SystemExit, cmdline.args_pathoc, [ + "pathoc", "--show-uas"], s, s) + + a = cmdline.args_pathoc(["pathoc", "foo.com:8888", "get:/"]) + assert a.port == 8888 + + a = cmdline.args_pathoc(["pathoc", "foo.com:xxx", "get:/"]) + assert perror.called + perror.reset_mock() + + a = cmdline.args_pathoc(["pathoc", "-I", "10, 20", "foo.com:8888", "get:/"]) + assert a.ignorecodes == [10, 20] + + a = cmdline.args_pathoc(["pathoc", "-I", "xx, 20", "foo.com:8888", "get:/"]) + assert perror.called + perror.reset_mock() + + a = cmdline.args_pathoc(["pathoc", "-c", "foo:10", "foo.com:8888", "get:/"]) + assert a.connect_to == ["foo", 10] + + a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2"]) + assert a.use_http2 == True + assert a.ssl == True + + a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2-skip-connection-preface"]) + assert a.use_http2 == True + assert a.ssl == True + assert a.http2_skip_connection_preface == True + + a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"]) + assert perror.called + perror.reset_mock() + + a = cmdline.args_pathoc( + ["pathoc", "-c", "foo:bar", "foo.com:8888", "get:/"]) + assert perror.called + perror.reset_mock() + + a = cmdline.args_pathoc( + [ + "pathoc", + "foo.com:8888", + tutils.test_data.path("data/request") + ] + ) + assert len(list(a.requests)) == 1 + + tutils.raises( + SystemExit, + cmdline.args_pathoc, + ["pathoc", "foo.com", "invalid"], + s, s + ) diff --git a/test/test_pathod.py b/test/test_pathod.py index f85ef38d0..1a3a5004d 100644 --- a/test/test_pathod.py +++ b/test/test_pathod.py @@ -1,7 +1,7 @@ import sys import cStringIO from libpathod import pathod, version -from netlib import tcp, http +from netlib import tcp, http, http2 import tutils @@ -269,3 +269,15 @@ class TestDaemonSSL(CommonTests): r, _ = self.pathoc([r"get:/p/202"]) assert r[0].status_code == 202 assert self.d.last_log()["cipher"][1] > 0 + +class TestHTTP2(tutils.DaemonTests): + force_http2 = True + ssl = True + noweb = True + noapi = True + nohang = True + + def test_http2(self): + r, _ = self.pathoc(["GET:/"], ssl=True, use_http2=True) + print(r) + assert r[0].status_code == "800" diff --git a/test/test_pathod_cmdline.py b/test/test_pathod_cmdline.py new file mode 100644 index 000000000..829c4b32e --- /dev/null +++ b/test/test_pathod_cmdline.py @@ -0,0 +1,85 @@ +from libpathod import pathod_cmdline as cmdline +import tutils +import cStringIO +import mock + + +@mock.patch("argparse.ArgumentParser.error") +def test_pathod(perror): + assert cmdline.args_pathod(["pathod"]) + + a = cmdline.args_pathod( + [ + "pathod", + "--cert", + tutils.test_data.path("data/testkey.pem") + ] + ) + assert a.ssl_certs + + a = cmdline.args_pathod( + [ + "pathod", + "--cert", + "nonexistent" + ] + ) + assert perror.called + perror.reset_mock() + + a = cmdline.args_pathod( + [ + "pathod", + "-a", + "foo=200" + ] + ) + assert a.anchors + + a = cmdline.args_pathod( + [ + "pathod", + "-a", + "foo=" + tutils.test_data.path("data/response") + ] + ) + assert a.anchors + + a = cmdline.args_pathod( + [ + "pathod", + "-a", + "?=200" + ] + ) + assert perror.called + perror.reset_mock() + + a = cmdline.args_pathod( + [ + "pathod", + "-a", + "foo" + ] + ) + assert perror.called + perror.reset_mock() + + a = cmdline.args_pathod( + [ + "pathod", + "--limit-size", + "200k" + ] + ) + assert a.sizelimit + + a = cmdline.args_pathod( + [ + "pathod", + "--limit-size", + "q" + ] + ) + assert perror.called + perror.reset_mock() diff --git a/test/tutils.py b/test/tutils.py index c56c60d43..2184ade56 100644 --- a/test/tutils.py +++ b/test/tutils.py @@ -27,7 +27,7 @@ class DaemonTests(object): klass.d = test.Daemon( staticdir=test_data.path("data"), anchors=[ - (re.compile("/anchor/.*"), language.parse_pathod("202:da")) + (re.compile("/anchor/.*"), "202:da") ], ssl = klass.ssl, ssloptions = so, @@ -73,7 +73,8 @@ class DaemonTests(object): timeout=None, connect_to=None, ssl=None, - ws_read_limit=None + ws_read_limit=None, + use_http2=False, ): """ Returns a (messages, text log) tuple. @@ -86,7 +87,8 @@ class DaemonTests(object): ssl=ssl, ws_read_limit=ws_read_limit, timeout = timeout, - fp = logfp + fp = logfp, + use_http2 = use_http2, ) c.connect(connect_to) ret = []