Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-27 02:24:18 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #27 from Kriechi/http2-wip

Browse Source 
HTTP/2: add initial support
This commit is contained in:
Aldo Cortesi 2015-06-18 16:16:40 +12:00
commit 274d0333f8
19 changed files with 1176 additions and 740 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

442
libpathod/cmdline.py
View File

@ -1,442 +0,0 @@
#!/usr/bin/env python
import sys
import argparse
import os
import os.path
import re
from netlib import http_uastrings
from . import pathoc, pathod, version, utils, language
def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
preparser = argparse.ArgumentParser(add_help=False)
preparser.add_argument(
"--show-uas", dest="showua", action="store_true", default=False,
help="Print user agent shortcuts and exit."
)
pa = preparser.parse_known_args(argv)[0]
if pa.showua:
print >> stdout, "User agent strings:"
for i in http_uastrings.UASTRINGS:
print >> stdout, " ", i[1], i[0]
sys.exit(0)
parser = argparse.ArgumentParser(
description='A perverse HTTP client.', parents=[preparser]
)
parser.add_argument(
'--version',
action='version',
version="pathoc " + version.VERSION
)
parser.add_argument(
"-c", dest="connect_to", type=str, default=False,
metavar = "HOST:PORT",
help="Issue an HTTP CONNECT to connect to the specified host."
)
parser.add_argument(
"--memo-limit", dest='memolimit', default=5000, type=int, metavar="N",
help='Stop if we do not find a valid request after N attempts.'
)
parser.add_argument(
"-m", dest='memo', action="store_true", default=False,
help="""
Remember specs, and never play the same one twice. Note that this
means requests have to be rendered in memory, which means that
large generated data can cause issues.
"""
)
parser.add_argument(
"-n", dest='repeat', default=1, type=int, metavar="N",
help='Repeat N times. If 0 repeat for ever.'
)
parser.add_argument(
"-w", dest='wait', default=0, type=float, metavar="N",
help='Wait N seconds between each request.'
)
parser.add_argument(
"-r", dest="random", action="store_true", default=False,
help="""
Select a random request from those specified. If this is not specified,
requests are all played in sequence.
"""
)
parser.add_argument(
"-t", dest="timeout", type=int, default=None,
help="Connection timeout"
)
parser.add_argument(
"--http2", dest="use_http2", action="store_true", default=False,
help='Perform all requests over a single HTTP/2 connection.'
)
parser.add_argument(
"--http2-skip-connection-preface",
dest="http2_skip_connection_preface",
action="store_true",
default=False,
help='Skips the HTTP/2 connection preface before sending requests.')
parser.add_argument(
'host', type=str,
metavar = "host[:port]",
help='Host and port to connect to'
)
parser.add_argument(
'requests', type=str, nargs="+",
help="""
Request specification, or path to a file containing request
specifcations
"""
)
group = parser.add_argument_group(
'SSL',
)
group.add_argument(
"-s", dest="ssl", action="store_true", default=False,
help="Connect with SSL"
)
group.add_argument(
"-C", dest="clientcert", type=str, default=False,
help="Path to a file containing client certificate and private key"
)
group.add_argument(
"-i", dest="sni", type=str, default=False,
help="SSL Server Name Indication"
)
group.add_argument(
"--ciphers", dest="ciphers", type=str, default=False,
help="SSL cipher specification"
)
group.add_argument(
"--sslversion", dest="sslversion", type=int, default=4,
choices=[1, 2, 3, 4],
help="""
Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
to SSLv23.
"""
)
group = parser.add_argument_group(
'Controlling Output',
"""
Some of these options expand generated values for logging - if
you're generating large data, use them with caution.
"""
)
group.add_argument(
"-I", dest="ignorecodes", type=str, default="",
help="Comma-separated list of response codes to ignore"
)
group.add_argument(
"-S", dest="showssl", action="store_true", default=False,
help="Show info on SSL connection"
)
group.add_argument(
"-e", dest="explain", action="store_true", default=False,
help="Explain requests"
)
group.add_argument(
"-o", dest="oneshot", action="store_true", default=False,
help="Oneshot - exit after first non-ignored response"
)
group.add_argument(
"-q", dest="showreq", action="store_true", default=False,
help="Print full request"
)
group.add_argument(
"-p", dest="showresp", action="store_true", default=False,
help="Print full response"
)
group.add_argument(
"-T", dest="ignoretimeout", action="store_true", default=False,
help="Ignore timeouts"
)
group.add_argument(
"-x", dest="hexdump", action="store_true", default=False,
help="Output in hexdump format"
)
args = parser.parse_args(argv[1:])
args.port = None
if ":" in args.host:
h, p = args.host.rsplit(":", 1)
try:
p = int(p)
except ValueError:
return parser.error("Invalid port in host spec: %s" % args.host)
args.host = h
args.port = p
if args.port is None:
args.port = 443 if args.ssl else 80
try:
args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i]
except ValueError:
return parser.error(
"Invalid return code specification: %s" %
args.ignorecodes)
if args.connect_to:
parts = args.connect_to.split(":")
if len(parts) != 2:
return parser.error(
"Invalid CONNECT specification: %s" %
args.connect_to)
try:
parts[1] = int(parts[1])
except ValueError:
return parser.error(
"Invalid CONNECT specification: %s" %
args.connect_to)
args.connect_to = parts
else:
args.connect_to = None
reqs = []
for r in args.requests:
if os.path.isfile(r):
data = open(r).read()
r = data
try:
reqs.append(language.parse_pathoc(r, args.use_http2))
except language.ParseException as v:
print >> stderr, "Error parsing request spec: %s" % v.msg
print >> stderr, v.marked()
sys.exit(1)
args.requests = reqs
return args
def go_pathoc(): # pragma: nocover
args = args_pathoc(sys.argv)
pathoc.main(args)
def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr):
parser = argparse.ArgumentParser(
description='A pathological HTTP/S daemon.'
)
parser.add_argument(
'--version',
action='version',
version="pathod " + version.VERSION
)
parser.add_argument(
"-p",
dest='port',
default=9999,
type=int,
help='Port. Specify 0 to pick an arbitrary empty port. (9999)'
)
parser.add_argument(
"-l",
dest='address',
default="127.0.0.1",
type=str,
help='Listening address. (127.0.0.1)'
)
parser.add_argument(
"-a",
dest='anchors',
default=[],
type=str,
action="append",
metavar="ANCHOR",
help="""
Add an anchor. Specified as a string with the form
pattern=spec or pattern=filepath, where pattern is a regular
expression.
"""
)
parser.add_argument(
"-c", dest='craftanchor', default=pathod.DEFAULT_ANCHOR, type=str,
help="""
Regular expression specifying anchor point for URL crafting
commands. (%s)
"""%pathod.DEFAULT_ANCHOR
)
parser.add_argument(
"--confdir",
action="store", type = str, dest="confdir", default='~/.mitmproxy',
help = "Configuration directory. (~/.mitmproxy)"
)
parser.add_argument(
"-d", dest='staticdir', default=None, type=str,
help='Directory for static files.'
)
parser.add_argument(
"-D", dest='daemonize', default=False, action="store_true",
help='Daemonize.'
)
parser.add_argument(
"-t", dest="timeout", type=int, default=None,
help="Connection timeout"
)
parser.add_argument(
"--limit-size",
dest='sizelimit',
default=None,
type=str,
help='Size limit of served responses. Understands size suffixes, i.e. 100k.')
parser.add_argument(
"--noapi", dest='noapi', default=False, action="store_true",
help='Disable API.'
)
parser.add_argument(
"--nohang", dest='nohang', default=False, action="store_true",
help='Disable pauses during crafted response generation.'
)
parser.add_argument(
"--noweb", dest='noweb', default=False, action="store_true",
help='Disable both web interface and API.'
)
parser.add_argument(
"--nocraft",
dest='nocraft',
default=False,
action="store_true",
help='Disable response crafting. If anchors are specified, they still work.')
parser.add_argument(
"--webdebug", dest='webdebug', default=False, action="store_true",
help='Debugging mode for the web app (dev only).'
)
group = parser.add_argument_group(
'SSL',
)
group.add_argument(
"-s", dest='ssl', default=False, action="store_true",
help='Run in HTTPS mode.'
)
group.add_argument(
"--cn",
dest="cn",
type=str,
default=None,
help="CN for generated SSL certs. Default: %s" %
pathod.DEFAULT_CERT_DOMAIN)
group.add_argument(
"-C", dest='ssl_not_after_connect', default=False, action="store_true",
help="Don't expect SSL after a CONNECT request."
)
group.add_argument(
"--cert", dest='ssl_certs', default=[], type=str,
metavar = "SPEC", action="append",
help = """
Add an SSL certificate. SPEC is of the form "[domain=]path". The domain
may include a wildcard, and is equal to "*" if not specified. The file
at path is a certificate in PEM format. If a private key is included in
the PEM, it is used, else the default key in the conf dir is used. Can
be passed multiple times.
"""
)
group.add_argument(
"--ciphers", dest="ciphers", type=str, default=False,
help="SSL cipher specification"
)
group.add_argument(
"--san", dest="sans", type=str, default=[], action="append",
metavar="SAN",
help="""
Subject Altnernate Name to add to the server certificate.
May be passed multiple times.
"""
)
group.add_argument(
"--sslversion", dest="sslversion", type=int, default=4,
choices=[1, 2, 3, 4],
help=""""Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
to SSLv23."""
)
group = parser.add_argument_group(
'Controlling Logging',
"""
Some of these options expand generated values for logging - if
you're generating large data, use them with caution.
"""
)
group.add_argument(
"-e", dest="explain", action="store_true", default=False,
help="Explain responses"
)
group.add_argument(
"-f", dest='logfile', default=None, type=str,
help='Log to file.'
)
group.add_argument(
"-q", dest="logreq", action="store_true", default=False,
help="Log full request"
)
group.add_argument(
"-r", dest="logresp", action="store_true", default=False,
help="Log full response"
)
group.add_argument(
"-x", dest="hexdump", action="store_true", default=False,
help="Log request/response in hexdump format"
)
args = parser.parse_args(argv[1:])
certs = []
for i in args.ssl_certs:
parts = i.split("=", 1)
if len(parts) == 1:
parts = ["*", parts[0]]
parts[1] = os.path.expanduser(parts[1])
if not os.path.isfile(parts[1]):
return parser.error(
"Certificate file does not exist: %s" %
parts[1])
certs.append(parts)
args.ssl_certs = certs
alst = []
for i in args.anchors:
parts = utils.parse_anchor_spec(i)
if not parts:
return parser.error("Invalid anchor specification: %s" % i)
alst.append(parts)
args.anchors = alst
sizelimit = None
if args.sizelimit:
try:
sizelimit = utils.parse_size(args.sizelimit)
except ValueError as v:
return parser.error(v)
args.sizelimit = sizelimit
try:
args.craftanchor = re.compile(args.craftanchor)
except re.error:
return parser.error(
"Invalid regex in craft anchor: %s" % args.craftanchor
)
anchors = []
for patt, spec in args.anchors:
if os.path.isfile(spec):
data = open(spec).read()
spec = data
try:
req = language.parse_pathod(spec)
except language.ParseException as v:
print >> stderr, "Error parsing anchor spec: %s" % v.msg
print >> stderr, v.marked()
sys.exit(1)
try:
arex = re.compile(patt)
except re.error:
return parser.error("Invalid regex in anchor: %s" % patt)
anchors.append((arex, req))
args.anchors = anchors
return args
def go_pathod(): # pragma: nocover
args = args_pathod(sys.argv)
pathod.main(args)

14
libpathod/language/__init__.py
View File

@ -19,7 +19,7 @@ def expand(msg):
yield msg yield msg
def parse_pathod(s): def parse_pathod(s, use_http2=False):
""" """
May raise ParseException May raise ParseException
""" """
@ -28,12 +28,17 @@ def parse_pathod(s):
except UnicodeError: except UnicodeError:
raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0) raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0)
try: try:
reqs = pp.Or( if use_http2:
[ expressions = [
# http2.Frame.expr(),
http2.Response.expr(),
]
else:
expressions = [
websockets.WebsocketFrame.expr(), websockets.WebsocketFrame.expr(),
http.Response.expr(), http.Response.expr(),
] ]
).parseString(s, parseAll=True) reqs = pp.Or(expressions).parseString(s, parseAll=True)
except pp.ParseException as v: except pp.ParseException as v:
raise exceptions.ParseException(v.msg, v.line, v.col) raise exceptions.ParseException(v.msg, v.line, v.col)
return itertools.chain(*[expand(i) for i in reqs]) return itertools.chain(*[expand(i) for i in reqs])
@ -55,7 +60,6 @@ def parse_pathoc(s, use_http2=False):
websockets.WebsocketClientFrame.expr(), websockets.WebsocketClientFrame.expr(),
http.Request.expr(), http.Request.expr(),
] ]
reqs = pp.OneOrMore(pp.Or(expressions)).parseString(s, parseAll=True) reqs = pp.OneOrMore(pp.Or(expressions)).parseString(s, parseAll=True)
except pp.ParseException as v: except pp.ParseException as v:
raise exceptions.ParseException(v.msg, v.line, v.col) raise exceptions.ParseException(v.msg, v.line, v.col)

6
libpathod/language/http.py
View File

@ -367,10 +367,6 @@ class Request(_HTTPMessage):
return ":".join([i.spec() for i in self.tokens]) return ":".join([i.spec() for i in self.tokens])
class PathodErrorResponse(Response):
pass
def make_error_response(reason, body=None): def make_error_response(reason, body=None):
tokens = [ tokens = [
Code("800"), Code("800"),
@ -381,4 +377,4 @@ def make_error_response(reason, body=None):
Reason(base.TokValueLiteral(reason)), Reason(base.TokValueLiteral(reason)),
Body(base.TokValueLiteral("pathod error: " + (body or reason))), Body(base.TokValueLiteral("pathod error: " + (body or reason))),
] ]
return PathodErrorResponse(tokens) return Response(tokens)

120
libpathod/language/http2.py
View File

@ -35,8 +35,15 @@ class Path(base.Value):
class Header(base.KeyValue): class Header(base.KeyValue):
unique_name = None
preamble = "h" preamble = "h"
def values(self, settings):
return (
self.key.get_generator(settings),
self.value.get_generator(settings),
)
class Body(base.Value): class Body(base.Value):
preamble = "b" preamble = "b"
@ -46,13 +53,21 @@ class Times(base.Integer):
preamble = "x" preamble = "x"
class Code(base.Integer):
pass
class Request(message.Message): class Request(message.Message):
comps = ( comps = (
Header, Header,
Body, Body,
Times, Times,
) )
logattrs = ["method", "path"]
def __init__(self, tokens):
super(Request, self).__init__(tokens)
self.rendered_values = None
@property @property
def method(self): def method(self):
@ -87,7 +102,6 @@ class Request(message.Message):
Method.expr(), Method.expr(),
base.Sep, base.Sep,
Path.expr(), Path.expr(),
base.Sep,
pp.ZeroOrMore(base.Sep + atom) pp.ZeroOrMore(base.Sep + atom)
] ]
) )
@ -95,25 +109,99 @@ class Request(message.Message):
return resp return resp
def resolve(self, settings, msg=None): def resolve(self, settings, msg=None):
tokens = self.tokens[:] return self
return self.__class__(
[i.resolve(settings, self) for i in tokens]
)
def values(self, settings): def values(self, settings):
return settings.protocol.create_request( if self.rendered_values:
self.method.value.get_generator(settings), return self.rendered_values
self.path, else:
self.headers, headers = [header.values(settings) for header in self.headers]
self.body)
body = self.body
if body:
body = body.string()
self.rendered_values = settings.protocol.create_request(
self.method.string(),
self.path.string(),
headers, # TODO: parse that into a dict?!
body)
return self.rendered_values
def spec(self): def spec(self):
return ":".join([i.spec() for i in self.tokens]) return ":".join([i.spec() for i in self.tokens])
# class H2F(base.CaselessLiteral): class Response(message.Message):
# TOK = "h2f" unique_name = None
# comps = (
# Header,
# class WebsocketFrame(message.Message): Body,
)
def __init__(self, tokens):
super(Response, self).__init__(tokens)
self.rendered_values = None
self.stream_id = 0
@property
def code(self):
return self.tok(Code)
@property
def headers(self):
return self.toks(Header)
@property
def body(self):
return self.tok(Body)
@property
def actions(self):
return []
def resolve(self, settings, msg=None):
return self
@classmethod
def expr(klass):
parts = [i.expr() for i in klass.comps]
atom = pp.MatchFirst(parts)
resp = pp.And(
[
Code.expr(),
pp.ZeroOrMore(base.Sep + atom)
]
)
resp = resp.setParseAction(klass)
return resp
def values(self, settings):
if self.rendered_values:
return self.rendered_values
else:
headers = [header.values(settings) for header in self.headers]
body = self.body
if body:
body = body.string()
self.rendered_values = settings.protocol.create_response(
self.code.string(),
self.stream_id,
headers, # TODO: parse that into a dict?!
body)
return self.rendered_values
def spec(self):
return ":".join([i.spec() for i in self.tokens])
def make_error_response(reason, body=None):
tokens = [
Code("800"),
Body(base.TokValueLiteral("pathod error: " + (body or reason))),
]
return Response(tokens)
# class Frame(message.Message):
# pass # pass

19
libpathod/pathoc.py
View File

@ -30,13 +30,8 @@ class SSLInfo:
self.certchain, self.cipher, self.alp = certchain, cipher, alp self.certchain, self.cipher, self.alp = certchain, cipher, alp
def __str__(self): def __str__(self):
if self.alp:
alp = self.alp
else:
alp = '<no protocol negotiated>'
parts = [ parts = [
"Application Layer Protocol: %s" % alp, "Application Layer Protocol: %s" % self.alp,
"Cipher: %s, %s bit, %s" % self.cipher, "Cipher: %s, %s bit, %s" % self.cipher,
"SSL certificate chain:" "SSL certificate chain:"
] ]
@ -155,13 +150,14 @@ class Pathoc(tcp.TCPClient):
# SSL # SSL
ssl=None, ssl=None,
sni=None, sni=None,
sslversion=4, sslversion='SSLv23',
clientcert=None, clientcert=None,
ciphers=None, ciphers=None,
# HTTP/2 # HTTP/2
use_http2=False, use_http2=False,
http2_skip_connection_preface=False, http2_skip_connection_preface=False,
http2_framedump = False,
# Websockets # Websockets
ws_read_limit = None, ws_read_limit = None,
@ -199,6 +195,7 @@ class Pathoc(tcp.TCPClient):
self.use_http2 = use_http2 self.use_http2 = use_http2
self.http2_skip_connection_preface = http2_skip_connection_preface self.http2_skip_connection_preface = http2_skip_connection_preface
self.http2_framedump = http2_framedump
self.ws_read_limit = ws_read_limit self.ws_read_limit = ws_read_limit
@ -216,6 +213,9 @@ class Pathoc(tcp.TCPClient):
self.ws_framereader = None self.ws_framereader = None
if self.use_http2: if self.use_http2:
if not OpenSSL._util.lib.Cryptography_HAS_ALPN: # pragma: nocover
print >> sys.stderr, "HTTP/2 requires ALPN support. Please use OpenSSL >= 1.0.2."
print >> sys.stderr, "Pathoc might not be working as expected without ALPN."
self.protocol = http2.HTTP2Protocol(self) self.protocol = http2.HTTP2Protocol(self)
else: else:
# TODO: create HTTP or Websockets protocol # TODO: create HTTP or Websockets protocol
@ -259,7 +259,7 @@ class Pathoc(tcp.TCPClient):
an HTTP CONNECT request. an HTTP CONNECT request.
""" """
if self.use_http2 and not self.ssl: if self.use_http2 and not self.ssl:
raise ValueError("HTTP2 without SSL is not supported.") raise NotImplementedError("HTTP2 without SSL is not supported.")
tcp.TCPClient.connect(self) tcp.TCPClient.connect(self)
@ -294,7 +294,7 @@ class Pathoc(tcp.TCPClient):
if self.use_http2: if self.use_http2:
self.protocol.check_alpn() self.protocol.check_alpn()
if not self.http2_skip_connection_preface: if not self.http2_skip_connection_preface:
self.protocol.perform_connection_preface() self.protocol.perform_client_connection_preface()
if self.timeout: if self.timeout:
self.settimeout(self.timeout) self.settimeout(self.timeout)
@ -462,6 +462,7 @@ def main(args): # pragma: nocover
ciphers = args.ciphers, ciphers = args.ciphers,
use_http2 = args.use_http2, use_http2 = args.use_http2,
http2_skip_connection_preface = args.http2_skip_connection_preface, http2_skip_connection_preface = args.http2_skip_connection_preface,
http2_framedump = args.http2_framedump,
showreq = args.showreq, showreq = args.showreq,
showresp = args.showresp, showresp = args.showresp,
explain = args.explain, explain = args.explain,

224
libpathod/pathoc_cmdline.py Normal file
View File

@ -0,0 +1,224 @@
#!/usr/bin/env python
import sys
import argparse
import os
import os.path
import re
from netlib import http_uastrings
from . import pathoc, pathod, version, utils, language
def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
preparser = argparse.ArgumentParser(add_help=False)
preparser.add_argument(
"--show-uas", dest="showua", action="store_true", default=False,
help="Print user agent shortcuts and exit."
)
pa = preparser.parse_known_args(argv)[0]
if pa.showua:
print >> stdout, "User agent strings:"
for i in http_uastrings.UASTRINGS:
print >> stdout, " ", i[1], i[0]
sys.exit(0)
parser = argparse.ArgumentParser(
description='A perverse HTTP client.', parents=[preparser]
)
parser.add_argument(
'--version',
action='version',
version="pathoc " + version.VERSION
)
parser.add_argument(
"-c", dest="connect_to", type=str, default=False,
metavar = "HOST:PORT",
help="Issue an HTTP CONNECT to connect to the specified host."
)
parser.add_argument(
"--memo-limit", dest='memolimit', default=5000, type=int, metavar="N",
help='Stop if we do not find a valid request after N attempts.'
)
parser.add_argument(
"-m", dest='memo', action="store_true", default=False,
help="""
Remember specs, and never play the same one twice. Note that this
means requests have to be rendered in memory, which means that
large generated data can cause issues.
"""
)
parser.add_argument(
"-n", dest='repeat', default=1, type=int, metavar="N",
help='Repeat N times. If 0 repeat for ever.'
)
parser.add_argument(
"-w", dest='wait', default=0, type=float, metavar="N",
help='Wait N seconds between each request.'
)
parser.add_argument(
"-r", dest="random", action="store_true", default=False,
help="""
Select a random request from those specified. If this is not specified,
requests are all played in sequence.
"""
)
parser.add_argument(
"-t", dest="timeout", type=int, default=None,
help="Connection timeout"
)
parser.add_argument(
"--http2", dest="use_http2", action="store_true", default=False,
help='Perform all requests over a single HTTP/2 connection.'
)
parser.add_argument(
"--http2-skip-connection-preface",
dest="http2_skip_connection_preface",
action="store_true",
default=False,
help='Skips the HTTP/2 connection preface before sending requests.')
parser.add_argument(
'host', type=str,
metavar = "host[:port]",
help='Host and port to connect to'
)
parser.add_argument(
'requests', type=str, nargs="+",
help="""
Request specification, or path to a file containing request
specifcations
"""
)
group = parser.add_argument_group(
'SSL',
)
group.add_argument(
"-s", dest="ssl", action="store_true", default=False,
help="Connect with SSL"
)
group.add_argument(
"-C", dest="clientcert", type=str, default=False,
help="Path to a file containing client certificate and private key"
)
group.add_argument(
"-i", dest="sni", type=str, default=False,
help="SSL Server Name Indication"
)
group.add_argument(
"--ciphers", dest="ciphers", type=str, default=False,
help="SSL cipher specification"
)
group.add_argument(
"--sslversion", dest="sslversion", type=str, default='SSLv23',
choices=utils.SSLVERSIONS.keys(),
help=""""
Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23.
Default to SSLv23."""
)
group = parser.add_argument_group(
'Controlling Output',
"""
Some of these options expand generated values for logging - if
you're generating large data, use them with caution.
"""
)
group.add_argument(
"-I", dest="ignorecodes", type=str, default="",
help="Comma-separated list of response codes to ignore"
)
group.add_argument(
"-S", dest="showssl", action="store_true", default=False,
help="Show info on SSL connection"
)
group.add_argument(
"-e", dest="explain", action="store_true", default=False,
help="Explain requests"
)
group.add_argument(
"-o", dest="oneshot", action="store_true", default=False,
help="Oneshot - exit after first non-ignored response"
)
group.add_argument(
"-q", dest="showreq", action="store_true", default=False,
help="Print full request"
)
group.add_argument(
"-p", dest="showresp", action="store_true", default=False,
help="Print full response"
)
group.add_argument(
"-T", dest="ignoretimeout", action="store_true", default=False,
help="Ignore timeouts"
)
group.add_argument(
"-x", dest="hexdump", action="store_true", default=False,
help="Output in hexdump format"
)
group.add_argument(
"--http2-framedump", dest="http2_framedump", action="store_true", default=False,
help="Output all received & sent HTTP/2 frames"
)
args = parser.parse_args(argv[1:])
args.port = None
if ":" in args.host:
h, p = args.host.rsplit(":", 1)
try:
p = int(p)
except ValueError:
return parser.error("Invalid port in host spec: %s" % args.host)
args.host = h
args.port = p
if args.port is None:
args.port = 443 if args.ssl else 80
try:
args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i]
except ValueError:
return parser.error(
"Invalid return code specification: %s" %
args.ignorecodes)
if args.connect_to:
parts = args.connect_to.split(":")
if len(parts) != 2:
return parser.error(
"Invalid CONNECT specification: %s" %
args.connect_to)
try:
parts[1] = int(parts[1])
except ValueError:
return parser.error(
"Invalid CONNECT specification: %s" %
args.connect_to)
args.connect_to = parts
else:
args.connect_to = None
if args.http2_skip_connection_preface:
args.use_http2 = True
if args.use_http2:
args.ssl = True
reqs = []
for r in args.requests:
if os.path.isfile(r):
data = open(r).read()
r = data
try:
reqs.append(language.parse_pathoc(r, args.use_http2))
except language.ParseException as v:
print >> stderr, "Error parsing request spec: %s" % v.msg
print >> stderr, v.marked()
sys.exit(1)
args.requests = reqs
return args
def go_pathoc(): # pragma: nocover
args = args_pathoc(sys.argv)
pathoc.main(args)

278
libpathod/pathod.py
View File

@ -7,7 +7,7 @@ import urllib
import re import re
import time import time
from netlib import tcp, http, wsgi, certutils, websockets from netlib import tcp, http, http2, wsgi, certutils, websockets, odict
from . import version, app, language, utils, log from . import version, app, language, utils, log
import language.http import language.http
@ -20,7 +20,7 @@ DEFAULT_CERT_DOMAIN = "pathod.net"
CONFDIR = "~/.mitmproxy" CONFDIR = "~/.mitmproxy"
CERTSTORE_BASENAME = "mitmproxy" CERTSTORE_BASENAME = "mitmproxy"
CA_CERT_NAME = "mitmproxy-ca.pem" CA_CERT_NAME = "mitmproxy-ca.pem"
DEFAULT_ANCHOR = r"/p/?" DEFAULT_CRAFT_ANCHOR = "/p/"
logger = logging.getLogger('pathod') logger = logging.getLogger('pathod')
@ -39,21 +39,23 @@ class SSLOptions:
request_client_cert=False, request_client_cert=False,
sslversion=tcp.SSLv23_METHOD, sslversion=tcp.SSLv23_METHOD,
ciphers=None, ciphers=None,
certs=None certs=None,
alpn_select=http2.HTTP2Protocol.ALPN_PROTO_H2,
): ):
self.confdir = confdir self.confdir = confdir
self.cn = cn self.cn = cn
self.sans = sans
self.not_after_connect = not_after_connect
self.request_client_cert = request_client_cert
self.sslversion = sslversion
self.ciphers = ciphers
self.alpn_select = alpn_select
self.certstore = certutils.CertStore.from_store( self.certstore = certutils.CertStore.from_store(
os.path.expanduser(confdir), os.path.expanduser(confdir),
CERTSTORE_BASENAME CERTSTORE_BASENAME
) )
for i in certs or []: for i in certs or []:
self.certstore.add_cert_file(*i) self.certstore.add_cert_file(*i)
self.not_after_connect = not_after_connect
self.request_client_cert = request_client_cert
self.ciphers = ciphers
self.sslversion = sslversion
self.sans = sans
def get_cert(self, name): def get_cert(self, name):
if self.cn: if self.cn:
@ -67,32 +69,37 @@ class PathodHandler(tcp.BaseHandler):
wbufsize = 0 wbufsize = 0
sni = None sni = None
def __init__(self, connection, address, server, logfp, settings): def __init__(self, connection, address, server, logfp, settings, http2_framedump=False):
self.logfp = logfp
tcp.BaseHandler.__init__(self, connection, address, server) tcp.BaseHandler.__init__(self, connection, address, server)
self.logfp = logfp
self.settings = copy.copy(settings) self.settings = copy.copy(settings)
self.protocol = None
self.use_http2 = False
self.http2_framedump = http2_framedump
def handle_sni(self, connection): def _handle_sni(self, connection):
self.sni = connection.get_servername() self.sni = connection.get_servername()
def http_serve_crafted(self, crafted): def http_serve_crafted(self, crafted):
"""
This method is HTTP/1 and HTTP/2 capable.
"""
error, crafted = self.server.check_policy( error, crafted = self.server.check_policy(
crafted, self.settings crafted, self.settings
) )
if error: if error:
err = language.http.make_error_response(error) err = self.make_http_error_response(error)
language.serve(err, self.wfile, self.settings) language.serve(err, self.wfile, self.settings)
return None, dict( return None, dict(
type="error", type="error",
msg = error msg = error
) )
if self.server.explain and not isinstance( if self.server.explain and not hasattr(crafted, 'is_error_response'):
crafted,
language.http.PathodErrorResponse
):
crafted = crafted.freeze(self.settings) crafted = crafted.freeze(self.settings)
log.write(self.logfp, ">> Spec: %s" % crafted.spec()) log.write(self.logfp, ">> Spec: %s" % crafted.spec())
response_log = language.serve( response_log = language.serve(
crafted, crafted,
self.wfile, self.wfile,
@ -152,6 +159,8 @@ class PathodHandler(tcp.BaseHandler):
def handle_http_connect(self, connect, lg): def handle_http_connect(self, connect, lg):
""" """
This method is HTTP/1 only.
Handle a CONNECT request. Handle a CONNECT request.
""" """
http.read_headers(self.rfile) http.read_headers(self.rfile)
@ -169,10 +178,11 @@ class PathodHandler(tcp.BaseHandler):
self.convert_to_ssl( self.convert_to_ssl(
cert, cert,
key, key,
handle_sni=self.handle_sni, handle_sni=self._handle_sni,
request_client_cert=self.server.ssloptions.request_client_cert, request_client_cert=self.server.ssloptions.request_client_cert,
cipher_list=self.server.ssloptions.ciphers, cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion, method=self.server.ssloptions.sslversion,
alpn_select=self.server.ssloptions.alpn_select,
) )
except tcp.NetLibError as v: except tcp.NetLibError as v:
s = str(v) s = str(v)
@ -182,10 +192,12 @@ class PathodHandler(tcp.BaseHandler):
def handle_http_app(self, method, path, headers, content, lg): def handle_http_app(self, method, path, headers, content, lg):
""" """
This method is HTTP/1 only.
Handle a request to the built-in app. Handle a request to the built-in app.
""" """
if self.server.noweb: if self.server.noweb:
crafted = language.http.make_error_response("Access Denied") crafted = self.make_http_error_response("Access Denied")
language.serve(crafted, self.wfile, self.settings) language.serve(crafted, self.wfile, self.settings)
return None, dict( return None, dict(
type="error", type="error",
@ -206,6 +218,8 @@ class PathodHandler(tcp.BaseHandler):
def handle_http_request(self): def handle_http_request(self):
""" """
This method is HTTP/1 and HTTP/2 capable.
Returns a (handler, log) tuple. Returns a (handler, log) tuple.
handler: Handler for the next request, or None to disconnect handler: Handler for the next request, or None to disconnect
@ -214,28 +228,26 @@ class PathodHandler(tcp.BaseHandler):
lr = self.rfile if self.server.logreq else None lr = self.rfile if self.server.logreq else None
lw = self.wfile if self.server.logresp else None lw = self.wfile if self.server.logresp else None
with log.Log(self.logfp, self.server.hexdump, lr, lw) as lg: with log.Log(self.logfp, self.server.hexdump, lr, lw) as lg:
line = http.get_request_line(self.rfile) if self.use_http2:
if not line: self.protocol.perform_server_connection_preface()
# Normal termination stream_id, headers, body = self.protocol.read_request()
return None, None method = headers[':method']
path = headers[':path']
m = utils.MemBool() headers = odict.ODict(headers)
if m(http.parse_init_connect(line)): httpversion = ""
return self.handle_http_connect(m.v, lg)
elif m(http.parse_init_proxy(line)):
method, _, _, _, path, httpversion = m.v
elif m(http.parse_init_http(line)):
method, path, httpversion = m.v
else: else:
s = "Invalid first line: %s" % repr(line) req = self.read_http_request(lg)
lg(s) if 'next_handle' in req:
return None, dict(type="error", msg=s) return req['next_handle']
if 'errors' in req:
headers = http.read_headers(self.rfile) return None, req['errors']
if headers is None: if not 'method' in req or not 'path' in req:
s = "Invalid headers" return None, None
lg(s) method = req['method']
return None, dict(type="error", msg=s) path = req['path']
headers = req['headers']
body = req['body']
httpversion = req['httpversion']
clientcert = None clientcert = None
if self.clientcert: if self.clientcert:
@ -265,16 +277,6 @@ class PathodHandler(tcp.BaseHandler):
if self.ssl_established: if self.ssl_established:
retlog["cipher"] = self.get_current_cipher() retlog["cipher"] = self.get_current_cipher()
try:
content = http.read_http_body(
self.rfile, headers, None,
method, None, True
)
except http.HttpError as s:
s = str(s)
lg(s)
return None, dict(type="error", msg=s)
m = utils.MemBool() m = utils.MemBool()
websocket_key = websockets.check_client_handshake(headers) websocket_key = websockets.check_client_handshake(headers)
self.settings.websocket_key = websocket_key self.settings.websocket_key = websocket_key
@ -285,27 +287,40 @@ class PathodHandler(tcp.BaseHandler):
anchor_gen = language.parse_pathod("ws") anchor_gen = language.parse_pathod("ws")
else: else:
anchor_gen = None anchor_gen = None
for i in self.server.anchors:
if i[0].match(path): for regex, spec in self.server.anchors:
anchor_gen = i[1] if regex.match(path):
anchor_gen = language.parse_pathod(spec, self.use_http2)
break break
else: else:
if m(self.server.craftanchor.match(path)): if m(path.startswith(self.server.craftanchor)):
spec = urllib.unquote(path)[len(m.v.group()):] spec = urllib.unquote(path)[len(self.server.craftanchor):]
if spec: if spec:
try: try:
anchor_gen = language.parse_pathod(spec) anchor_gen = language.parse_pathod(spec, self.use_http2)
except language.ParseException as v: except language.ParseException as v:
lg("Parse error: %s" % v.msg) lg("Parse error: %s" % v.msg)
anchor_gen = iter([language.http.make_error_response( anchor_gen = iter([self.make_http_error_response(
"Parse Error", "Parse Error",
"Error parsing response spec: %s\n" % ( "Error parsing response spec: %s\n" % (
v.msg + v.marked() v.msg + v.marked()
) )
)]) )])
else:
if self.use_http2:
anchor_gen = iter([self.make_http_error_response(
"Spec Error",
"HTTP/2 only supports request/response with the craft anchor point: %s" %
self.server.craftanchor
)])
if anchor_gen: if anchor_gen:
spec = anchor_gen.next() spec = anchor_gen.next()
if self.use_http2 and isinstance(spec, language.http2.Response):
spec.stream_id = stream_id
lg("crafting spec: %s" % spec) lg("crafting spec: %s" % spec)
nexthandler, retlog["response"] = self.http_serve_crafted( nexthandler, retlog["response"] = self.http_serve_crafted(
spec spec
@ -315,7 +330,113 @@ class PathodHandler(tcp.BaseHandler):
else: else:
return nexthandler, retlog return nexthandler, retlog
else: else:
return self.handle_http_app(method, path, headers, content, lg) return self.handle_http_app(method, path, headers, body, lg)
def read_http_request(self, lg):
"""
This method is HTTP/1 only.
"""
line = http.get_request_line(self.rfile)
if not line:
# Normal termination
return dict()
m = utils.MemBool()
if m(http.parse_init_connect(line)):
return dict(next_handle=self.handle_http_connect(m.v, lg))
elif m(http.parse_init_proxy(line)):
method, _, _, _, path, httpversion = m.v
elif m(http.parse_init_http(line)):
method, path, httpversion = m.v
else:
s = "Invalid first line: %s" % repr(line)
lg(s)
return dict(errors=dict(type="error", msg=s))
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
lg(s)
return dict(errors=dict(type="error", msg=s))
try:
body = http.read_http_body(
self.rfile,
headers,
None,
method,
None,
True,
)
except http.HttpError as s:
s = str(s)
lg(s)
return dict(errors=dict(type="error", msg=s))
return dict(
method=method,
path=path,
headers=headers,
body=body,
httpversion=httpversion)
def make_http_error_response(self, reason, body=None):
"""
This method is HTTP/1 and HTTP/2 capable.
"""
if self.use_http2:
resp = language.http2.make_error_response(reason, body)
else:
resp = language.http.make_error_response(reason, body)
resp.is_error_response = True
return resp
def handle(self):
self.settimeout(self.server.timeout)
if self.server.ssl:
try:
cert, key, _ = self.server.ssloptions.get_cert(None)
self.convert_to_ssl(
cert,
key,
dhparams=self.server.ssloptions.certstore.dhparams,
handle_sni=self._handle_sni,
request_client_cert=self.server.ssloptions.request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion,
alpn_select=self.server.ssloptions.alpn_select,
)
except tcp.NetLibError as v:
s = str(v)
self.server.add_log(
dict(
type="error",
msg=s
)
)
log.write(self.logfp, s)
return
alp = self.get_alpn_proto_negotiated()
if alp == http2.HTTP2Protocol.ALPN_PROTO_H2:
self.protocol = http2.HTTP2Protocol(self, is_server=True, dump_frames=self.http2_framedump)
self.use_http2 = True
# if not self.protocol:
# # TODO: create HTTP or Websockets protocol
# self.protocol = None
self.settings.protocol = self.protocol
handler = self.handle_http_request
while not self.finished:
handler, l = handler()
if l:
self.addlog(l)
if not handler:
return
def addlog(self, log): def addlog(self, log):
# FIXME: The bytes in the log should not be escaped. We do this at the # FIXME: The bytes in the log should not be escaped. We do this at the
@ -329,37 +450,6 @@ class PathodHandler(tcp.BaseHandler):
log["response_bytes"] = bytes log["response_bytes"] = bytes
self.server.add_log(log) self.server.add_log(log)
def handle(self):
if self.server.ssl:
try:
cert, key, _ = self.server.ssloptions.get_cert(None)
self.convert_to_ssl(
cert,
key,
handle_sni=self.handle_sni,
request_client_cert=self.server.ssloptions.request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion,
)
except tcp.NetLibError as v:
s = str(v)
self.server.add_log(
dict(
type="error",
msg=s
)
)
log.write(self.logfp, s)
return
self.settimeout(self.server.timeout)
handler = self.handle_http_request
while not self.finished:
handler, l = handler()
if l:
self.addlog(l)
if not handler:
return
class Pathod(tcp.TCPServer): class Pathod(tcp.TCPServer):
LOGBUF = 500 LOGBUF = 500
@ -369,7 +459,7 @@ class Pathod(tcp.TCPServer):
addr, addr,
ssl=False, ssl=False,
ssloptions=None, ssloptions=None,
craftanchor=re.compile(DEFAULT_ANCHOR), craftanchor=DEFAULT_CRAFT_ANCHOR,
staticdir=None, staticdir=None,
anchors=(), anchors=(),
sizelimit=None, sizelimit=None,
@ -382,6 +472,7 @@ class Pathod(tcp.TCPServer):
logresp=False, logresp=False,
explain=False, explain=False,
hexdump=False, hexdump=False,
http2_framedump=False,
webdebug=False, webdebug=False,
logfp=sys.stdout, logfp=sys.stdout,
): ):
@ -389,7 +480,7 @@ class Pathod(tcp.TCPServer):
addr: (address, port) tuple. If port is 0, a free port will be addr: (address, port) tuple. If port is 0, a free port will be
automatically chosen. automatically chosen.
ssloptions: an SSLOptions object. ssloptions: an SSLOptions object.
craftanchor: string specifying the path under which to anchor craftanchor: URL prefix specifying the path under which to anchor
response generation. response generation.
staticdir: path to a directory of static resources, or None. staticdir: path to a directory of static resources, or None.
anchors: List of (regex object, language.Request object) tuples, or anchors: List of (regex object, language.Request object) tuples, or
@ -409,6 +500,7 @@ class Pathod(tcp.TCPServer):
self.noapi, self.nohang = noapi, nohang self.noapi, self.nohang = noapi, nohang
self.timeout, self.logreq = timeout, logreq self.timeout, self.logreq = timeout, logreq
self.logresp, self.hexdump = logresp, hexdump self.logresp, self.hexdump = logresp, hexdump
self.http2_framedump = http2_framedump
self.explain = explain self.explain = explain
self.logfp = logfp self.logfp = logfp
@ -446,7 +538,8 @@ class Pathod(tcp.TCPServer):
client_address, client_address,
self, self,
self.logfp, self.logfp,
self.settings self.settings,
self.http2_framedump,
) )
try: try:
h.handle() h.handle()
@ -502,7 +595,7 @@ def main(args): # pragma: nocover
ciphers = args.ciphers, ciphers = args.ciphers,
sslversion = utils.SSLVERSIONS[args.sslversion], sslversion = utils.SSLVERSIONS[args.sslversion],
certs = args.ssl_certs, certs = args.ssl_certs,
sans = args.sans sans = args.sans,
) )
root = logging.getLogger() root = logging.getLogger()
@ -542,6 +635,7 @@ def main(args): # pragma: nocover
logreq = args.logreq, logreq = args.logreq,
logresp = args.logresp, logresp = args.logresp,
hexdump = args.hexdump, hexdump = args.hexdump,
http2_framedump = args.http2_framedump,
explain = args.explain, explain = args.explain,
webdebug = args.webdebug webdebug = args.webdebug
) )

229
libpathod/pathod_cmdline.py Normal file
View File

@ -0,0 +1,229 @@
#!/usr/bin/env python
import sys
import argparse
import os
import os.path
import re
from netlib import http_uastrings
from . import pathoc, pathod, version, utils, language
def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr):
parser = argparse.ArgumentParser(
description='A pathological HTTP/S daemon.'
)
parser.add_argument(
'--version',
action='version',
version="pathod " + version.VERSION
)
parser.add_argument(
"-p",
dest='port',
default=9999,
type=int,
help='Port. Specify 0 to pick an arbitrary empty port. (9999)'
)
parser.add_argument(
"-l",
dest='address',
default="127.0.0.1",
type=str,
help='Listening address. (127.0.0.1)'
)
parser.add_argument(
"-a",
dest='anchors',
default=[],
type=str,
action="append",
metavar="ANCHOR",
help="""
Add an anchor. Specified as a string with the form
pattern=spec or pattern=filepath, where pattern is a regular
expression.
"""
)
parser.add_argument(
"-c", dest='craftanchor', default=pathod.DEFAULT_CRAFT_ANCHOR, type=str,
help="""
URL path specifying prefix for URL crafting
commands. (%s)
"""%pathod.DEFAULT_CRAFT_ANCHOR
)
parser.add_argument(
"--confdir",
action="store", type = str, dest="confdir", default='~/.mitmproxy',
help = "Configuration directory. (~/.mitmproxy)"
)
parser.add_argument(
"-d", dest='staticdir', default=None, type=str,
help='Directory for static files.'
)
parser.add_argument(
"-D", dest='daemonize', default=False, action="store_true",
help='Daemonize.'
)
parser.add_argument(
"-t", dest="timeout", type=int, default=None,
help="Connection timeout"
)
parser.add_argument(
"--limit-size",
dest='sizelimit',
default=None,
type=str,
help='Size limit of served responses. Understands size suffixes, i.e. 100k.')
parser.add_argument(
"--noapi", dest='noapi', default=False, action="store_true",
help='Disable API.'
)
parser.add_argument(
"--nohang", dest='nohang', default=False, action="store_true",
help='Disable pauses during crafted response generation.'
)
parser.add_argument(
"--noweb", dest='noweb', default=False, action="store_true",
help='Disable both web interface and API.'
)
parser.add_argument(
"--nocraft",
dest='nocraft',
default=False,
action="store_true",
help='Disable response crafting. If anchors are specified, they still work.')
parser.add_argument(
"--webdebug", dest='webdebug', default=False, action="store_true",
help='Debugging mode for the web app (dev only).'
)
group = parser.add_argument_group(
'SSL',
)
group.add_argument(
"-s", dest='ssl', default=False, action="store_true",
help='Run in HTTPS mode.'
)
group.add_argument(
"--cn",
dest="cn",
type=str,
default=None,
help="CN for generated SSL certs. Default: %s" %
pathod.DEFAULT_CERT_DOMAIN)
group.add_argument(
"-C", dest='ssl_not_after_connect', default=False, action="store_true",
help="Don't expect SSL after a CONNECT request."
)
group.add_argument(
"--cert", dest='ssl_certs', default=[], type=str,
metavar = "SPEC", action="append",
help = """
Add an SSL certificate. SPEC is of the form "[domain=]path". The domain
may include a wildcard, and is equal to "*" if not specified. The file
at path is a certificate in PEM format. If a private key is included in
the PEM, it is used, else the default key in the conf dir is used. Can
be passed multiple times.
"""
)
group.add_argument(
"--ciphers", dest="ciphers", type=str, default=False,
help="SSL cipher specification"
)
group.add_argument(
"--san", dest="sans", type=str, default=[], action="append",
metavar="SAN",
help="""
Subject Altnernate Name to add to the server certificate.
May be passed multiple times.
"""
)
group.add_argument(
"--sslversion", dest="sslversion", type=str, default='SSLv23',
choices=utils.SSLVERSIONS.keys(),
help=""""
Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23.
Default to SSLv23."""
)
group = parser.add_argument_group(
'Controlling Logging',
"""
Some of these options expand generated values for logging - if
you're generating large data, use them with caution.
"""
)
group.add_argument(
"-e", dest="explain", action="store_true", default=False,
help="Explain responses"
)
group.add_argument(
"-f", dest='logfile', default=None, type=str,
help='Log to file.'
)
group.add_argument(
"-q", dest="logreq", action="store_true", default=False,
help="Log full request"
)
group.add_argument(
"-r", dest="logresp", action="store_true", default=False,
help="Log full response"
)
group.add_argument(
"-x", dest="hexdump", action="store_true", default=False,
help="Log request/response in hexdump format"
)
group.add_argument(
"--http2-framedump", dest="http2_framedump", action="store_true", default=False,
help="Output all received & sent HTTP/2 frames"
)
args = parser.parse_args(argv[1:])
certs = []
for i in args.ssl_certs:
parts = i.split("=", 1)
if len(parts) == 1:
parts = ["*", parts[0]]
parts[1] = os.path.expanduser(parts[1])
if not os.path.isfile(parts[1]):
return parser.error(
"Certificate file does not exist: %s" %
parts[1])
certs.append(parts)
args.ssl_certs = certs
alst = []
for i in args.anchors:
parts = utils.parse_anchor_spec(i)
if not parts:
return parser.error("Invalid anchor specification: %s" % i)
alst.append(parts)
args.anchors = alst
sizelimit = None
if args.sizelimit:
try:
sizelimit = utils.parse_size(args.sizelimit)
except ValueError as v:
return parser.error(v)
args.sizelimit = sizelimit
anchors = []
for patt, spec in args.anchors:
if os.path.isfile(spec):
data = open(spec).read()
spec = data
try:
arex = re.compile(patt)
except re.error:
return parser.error("Invalid regex in anchor: %s" % patt)
anchors.append((arex, spec))
args.anchors = anchors
return args
def go_pathod(): # pragma: nocover
args = args_pathod(sys.argv)
pathod.main(args)

10
libpathod/utils.py
View File

@ -3,10 +3,12 @@ import sys
from netlib import tcp from netlib import tcp
SSLVERSIONS = { SSLVERSIONS = {
1: tcp.TLSv1_METHOD, 'TLSv1.2': tcp.TLSv1_2_METHOD,
2: tcp.SSLv2_METHOD, 'TLSv1.1': tcp.TLSv1_1_METHOD,
3: tcp.SSLv3_METHOD, 'TLSv1': tcp.TLSv1_METHOD,
4: tcp.SSLv23_METHOD, 'SSLv3': tcp.SSLv3_METHOD,
'SSLv2': tcp.SSLv2_METHOD,
'SSLv23': tcp.SSLv23_METHOD,
} }
SIZE_UNITS = dict( SIZE_UNITS = dict(

3
pathoc
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
from libpathod import cmdline
from libpathod import pathoc_cmdline as cmdline
if __name__ == "__main__": if __name__ == "__main__":
cmdline.go_pathoc() cmdline.go_pathoc()

3
pathod
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python #!/usr/bin/env python
from libpathod import cmdline
from libpathod import pathod_cmdline as cmdline
if __name__ == "__main__": if __name__ == "__main__":
cmdline.go_pathod() cmdline.go_pathod()

4
setup.py
View File

@ -40,8 +40,8 @@ setup(
include_package_data=True, include_package_data=True,
entry_points={ entry_points={
'console_scripts': [ 'console_scripts': [
"pathod = libpathod.cmdline:go_pathod", "pathod = libpathod.pathod_cmdline:go_pathod",
"pathoc = libpathod.cmdline:go_pathoc" "pathoc = libpathod.pathoc_cmdline:go_pathoc"
] ]
}, },
install_requires=[ install_requires=[

155
test/test_cmdline.py
View File

@ -1,155 +0,0 @@
from libpathod import cmdline
import tutils
import cStringIO
import mock
@mock.patch("argparse.ArgumentParser.error")
def test_pathod(perror):
assert cmdline.args_pathod(["pathod"])
a = cmdline.args_pathod(
[
"pathod",
"--cert",
tutils.test_data.path("data/testkey.pem")
]
)
assert a.ssl_certs
a = cmdline.args_pathod(
[
"pathod",
"--cert",
"nonexistent"
]
)
assert perror.called
perror.reset_mock()
a = cmdline.args_pathod(
[
"pathod",
"-a",
"foo=200"
]
)
assert a.anchors
a = cmdline.args_pathod(
[
"pathod",
"-a",
"foo=" + tutils.test_data.path("data/response")
]
)
assert a.anchors
a = cmdline.args_pathod(
[
"pathod",
"-a",
"?=200"
]
)
assert perror.called
perror.reset_mock()
a = cmdline.args_pathod(
[
"pathod",
"-a",
"foo"
]
)
assert perror.called
perror.reset_mock()
s = cStringIO.StringIO()
tutils.raises(
SystemExit,
cmdline.args_pathod,
["pathod", "-a", "foo=."],
s,
s
)
a = cmdline.args_pathod(
[
"pathod",
"--limit-size",
"200k"
]
)
assert a.sizelimit
a = cmdline.args_pathod(
[
"pathod",
"--limit-size",
"q"
]
)
assert perror.called
perror.reset_mock()
@mock.patch("argparse.ArgumentParser.error")
def test_pathoc(perror):
assert cmdline.args_pathoc(["pathoc", "foo.com", "get:/"])
s = cStringIO.StringIO()
tutils.raises(
SystemExit, cmdline.args_pathoc, [
"pathoc", "--show-uas"], s, s)
a = cmdline.args_pathoc(["pathoc", "foo.com:8888", "get:/"])
assert a.port == 8888
a = cmdline.args_pathoc(["pathoc", "foo.com:xxx", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(["pathoc", "-I", "10, 20", "foo.com:8888", "get:/"])
assert a.ignorecodes == [10, 20]
a = cmdline.args_pathoc(["pathoc", "-I", "xx, 20", "foo.com:8888", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(["pathoc", "-c", "foo:10", "foo.com:8888", "get:/"])
assert a.connect_to == ["foo", 10]
a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(
["pathoc", "-c", "foo:bar", "foo.com:8888", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(
[
"pathoc",
"foo.com:8888",
tutils.test_data.path("data/request")
]
)
assert len(list(a.requests)) == 1
a = cmdline.args_pathod(
[
"pathod",
"-c",
"?"
]
)
assert perror.called
perror.reset_mock()
tutils.raises(
SystemExit,
cmdline.args_pathoc,
["pathoc", "foo.com", "invalid"],
s, s
)

177
test/test_language_http2.py Normal file
View File

@ -0,0 +1,177 @@
import cStringIO
from netlib import tcp
from libpathod import language
from libpathod.language import http2, base
import netlib
import tutils
def parse_request(s):
return language.parse_pathoc(s, True).next()
def parse_response(s):
return language.parse_pathod(s, True).next()
def default_settings():
return language.Settings(
request_host = "foo.com",
protocol = netlib.http2.HTTP2Protocol(tcp.TCPClient(('localhost', 1234)))
)
def test_make_error_response():
d = cStringIO.StringIO()
s = http2.make_error_response("foo", "bar")
language.serve(s, d, default_settings())
class TestRequest:
def test_cached_values(self):
req = parse_request("get:/")
req_id = id(req)
assert req_id == id(req.resolve(default_settings()))
assert req.values(default_settings()) == req.values(default_settings())
def test_nonascii(self):
tutils.raises("ascii", parse_request, "get:\xf0")
def test_err(self):
tutils.raises(language.ParseException, parse_request, 'GET')
def test_simple(self):
r = parse_request('GET:"/foo"')
assert r.method.string() == "GET"
assert r.path.string() == "/foo"
r = parse_request('GET:/foo')
assert r.path.string() == "/foo"
def test_multiple(self):
r = list(language.parse_pathoc("GET:/ PUT:/"))
assert r[0].method.string() == "GET"
assert r[1].method.string() == "PUT"
assert len(r) == 2
l = """
GET
"/foo"
PUT
"/foo
bar"
"""
r = list(language.parse_pathoc(l, True))
assert len(r) == 2
assert r[0].method.string() == "GET"
assert r[1].method.string() == "PUT"
l = """
get:"http://localhost:9999/p/200"
get:"http://localhost:9999/p/200"
"""
r = list(language.parse_pathoc(l, True))
assert len(r) == 2
assert r[0].method.string() == "GET"
assert r[1].method.string() == "GET"
def test_render_simple(self):
s = cStringIO.StringIO()
r = parse_request("GET:'/foo'")
assert language.serve(
r,
s,
default_settings(),
)
def test_render_with_headers(self):
s = cStringIO.StringIO()
r = parse_request('GET:/foo:h"foo"="bar"')
assert language.serve(
r,
s,
default_settings(),
)
def test_render_with_body(self):
s = cStringIO.StringIO()
r = parse_request("GET:'/foo':bfoobar")
assert language.serve(
r,
s,
default_settings(),
)
def test_spec(self):
def rt(s):
s = parse_request(s).spec()
assert parse_request(s).spec() == s
rt("get:/foo")
class TestResponse:
def test_cached_values(self):
res = parse_response("200")
res_id = id(res)
assert res_id == id(res.resolve(default_settings()))
assert res.values(default_settings()) == res.values(default_settings())
def test_nonascii(self):
tutils.raises("ascii", parse_response, "200:\xf0")
def test_err(self):
tutils.raises(language.ParseException, parse_response, 'GET:/')
def test_simple(self):
r = parse_response('200')
assert r.code.string() == "200"
assert len(r.headers) == 0
r = parse_response('200:h"foo"="bar"')
assert r.code.string() == "200"
assert len(r.headers) == 1
assert r.headers[0].values(default_settings()) == ("foo", "bar")
assert r.body == None
r = parse_response('200:h"foo"="bar":bfoobar:h"bla"="fasel"')
assert r.code.string() == "200"
assert len(r.headers) == 2
assert r.headers[0].values(default_settings()) == ("foo", "bar")
assert r.headers[1].values(default_settings()) == ("bla", "fasel")
assert r.body.string() == "foobar"
def test_render_simple(self):
s = cStringIO.StringIO()
r = parse_response('200')
assert language.serve(
r,
s,
default_settings(),
)
def test_render_with_headers(self):
s = cStringIO.StringIO()
r = parse_response('200:h"foo"="bar"')
assert language.serve(
r,
s,
default_settings(),
)
def test_render_with_body(self):
s = cStringIO.StringIO()
r = parse_response('200:bfoobar')
assert language.serve(
r,
s,
default_settings(),
)
def test_spec(self):
def rt(s):
s = parse_response(s).spec()
assert parse_response(s).spec() == s
rt("200:bfoobar")

61
test/test_pathoc.py
View File

@ -1,8 +1,9 @@
import json import json
import cStringIO import cStringIO
import re import re
from mock import Mock
from netlib import tcp, http from netlib import tcp, http, http2
from libpathod import pathoc, test, version, pathod, language from libpathod import pathoc, test, version, pathod, language
import tutils import tutils
@ -22,7 +23,7 @@ class _TestDaemon:
ssloptions = self.ssloptions, ssloptions = self.ssloptions,
staticdir = tutils.test_data.path("data"), staticdir = tutils.test_data.path("data"),
anchors = [ anchors = [
(re.compile("/anchor/.*"), language.parse_pathod("202")) (re.compile("/anchor/.*"), "202")
] ]
) )
@ -86,8 +87,9 @@ class _TestDaemon:
class TestDaemonSSL(_TestDaemon): class TestDaemonSSL(_TestDaemon):
ssl = True ssl = True
ssloptions = pathod.SSLOptions( ssloptions = pathod.SSLOptions(
request_client_cert=True, request_client_cert = True,
sans = ["test1.com", "test2.com"] sans = ["test1.com", "test2.com"],
alpn_select = http2.HTTP2Protocol.ALPN_PROTO_H2,
) )
def test_sni(self): def test_sni(self):
@ -119,6 +121,14 @@ class TestDaemonSSL(_TestDaemon):
d = json.loads(r.content) d = json.loads(r.content)
assert d["log"][0]["request"]["clientcert"]["keyinfo"] assert d["log"][0]["request"]["clientcert"]["keyinfo"]
def test_http2_without_ssl(self):
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
use_http2 = True,
ssl = False,
)
tutils.raises(NotImplementedError, c.connect)
class TestDaemon(_TestDaemon): class TestDaemon(_TestDaemon):
ssl = False ssl = False
@ -216,3 +226,46 @@ class TestDaemon(_TestDaemon):
"HTTP/1.1 200 OK\r\n" "HTTP/1.1 200 OK\r\n"
) )
c.http_connect(to) c.http_connect(to)
class TestDaemonHTTP2(_TestDaemon):
ssl = True
def test_http2(self):
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
use_http2 = True,
ssl = True,
)
assert isinstance(c.protocol, http2.HTTP2Protocol)
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
)
assert c.protocol == None # TODO: change if other protocols get implemented
def test_http2_alpn(self):
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
ssl = True,
use_http2 = True,
http2_skip_connection_preface = True,
)
tmp_convert_to_ssl = c.convert_to_ssl
c.convert_to_ssl = Mock()
c.convert_to_ssl.side_effect = tmp_convert_to_ssl
c.connect()
_, kwargs = c.convert_to_ssl.call_args
assert set(kwargs['alpn_protos']) == set([b'http1.1', b'h2'])
def test_request(self):
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
ssl = True,
use_http2 = True,
)
c.connect()
resp = c.request("get:/p/200")
assert resp.status_code == "200"

64
test/test_pathoc_cmdline.py Normal file
View File

@ -0,0 +1,64 @@
from libpathod import pathoc_cmdline as cmdline
import tutils
import cStringIO
import mock
@mock.patch("argparse.ArgumentParser.error")
def test_pathoc(perror):
assert cmdline.args_pathoc(["pathoc", "foo.com", "get:/"])
s = cStringIO.StringIO()
tutils.raises(
SystemExit, cmdline.args_pathoc, [
"pathoc", "--show-uas"], s, s)
a = cmdline.args_pathoc(["pathoc", "foo.com:8888", "get:/"])
assert a.port == 8888
a = cmdline.args_pathoc(["pathoc", "foo.com:xxx", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(["pathoc", "-I", "10, 20", "foo.com:8888", "get:/"])
assert a.ignorecodes == [10, 20]
a = cmdline.args_pathoc(["pathoc", "-I", "xx, 20", "foo.com:8888", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(["pathoc", "-c", "foo:10", "foo.com:8888", "get:/"])
assert a.connect_to == ["foo", 10]
a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2"])
assert a.use_http2 == True
assert a.ssl == True
a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2-skip-connection-preface"])
assert a.use_http2 == True
assert a.ssl == True
assert a.http2_skip_connection_preface == True
a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(
["pathoc", "-c", "foo:bar", "foo.com:8888", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(
[
"pathoc",
"foo.com:8888",
tutils.test_data.path("data/request")
]
)
assert len(list(a.requests)) == 1
tutils.raises(
SystemExit,
cmdline.args_pathoc,
["pathoc", "foo.com", "invalid"],
s, s
)

14
test/test_pathod.py
View File

@ -1,7 +1,7 @@
import sys import sys
import cStringIO import cStringIO
from libpathod import pathod, version from libpathod import pathod, version
from netlib import tcp, http from netlib import tcp, http, http2
import tutils import tutils
@ -269,3 +269,15 @@ class TestDaemonSSL(CommonTests):
r, _ = self.pathoc([r"get:/p/202"]) r, _ = self.pathoc([r"get:/p/202"])
assert r[0].status_code == 202 assert r[0].status_code == 202
assert self.d.last_log()["cipher"][1] > 0 assert self.d.last_log()["cipher"][1] > 0
class TestHTTP2(tutils.DaemonTests):
force_http2 = True
ssl = True
noweb = True
noapi = True
nohang = True
def test_http2(self):
r, _ = self.pathoc(["GET:/"], ssl=True, use_http2=True)
print(r)
assert r[0].status_code == "800"

85
test/test_pathod_cmdline.py Normal file
View File

@ -0,0 +1,85 @@
from libpathod import pathod_cmdline as cmdline
import tutils
import cStringIO
import mock
@mock.patch("argparse.ArgumentParser.error")
def test_pathod(perror):
assert cmdline.args_pathod(["pathod"])
a = cmdline.args_pathod(
[
"pathod",
"--cert",
tutils.test_data.path("data/testkey.pem")
]
)
assert a.ssl_certs
a = cmdline.args_pathod(
[
"pathod",
"--cert",
"nonexistent"
]
)
assert perror.called
perror.reset_mock()
a = cmdline.args_pathod(
[
"pathod",
"-a",
"foo=200"
]
)
assert a.anchors
a = cmdline.args_pathod(
[
"pathod",
"-a",
"foo=" + tutils.test_data.path("data/response")
]
)
assert a.anchors
a = cmdline.args_pathod(
[
"pathod",
"-a",
"?=200"
]
)
assert perror.called
perror.reset_mock()
a = cmdline.args_pathod(
[
"pathod",
"-a",
"foo"
]
)
assert perror.called
perror.reset_mock()
a = cmdline.args_pathod(
[
"pathod",
"--limit-size",
"200k"
]
)
assert a.sizelimit
a = cmdline.args_pathod(
[
"pathod",
"--limit-size",
"q"
]
)
assert perror.called
perror.reset_mock()

8
test/tutils.py
View File

@ -27,7 +27,7 @@ class DaemonTests(object):
klass.d = test.Daemon( klass.d = test.Daemon(
staticdir=test_data.path("data"), staticdir=test_data.path("data"),
anchors=[ anchors=[
(re.compile("/anchor/.*"), language.parse_pathod("202:da")) (re.compile("/anchor/.*"), "202:da")
], ],
ssl = klass.ssl, ssl = klass.ssl,
ssloptions = so, ssloptions = so,
@ -73,7 +73,8 @@ class DaemonTests(object):
timeout=None, timeout=None,
connect_to=None, connect_to=None,
ssl=None, ssl=None,
ws_read_limit=None ws_read_limit=None,
use_http2=False,
): ):
""" """
Returns a (messages, text log) tuple. Returns a (messages, text log) tuple.
@ -86,7 +87,8 @@ class DaemonTests(object):
ssl=ssl, ssl=ssl,
ws_read_limit=ws_read_limit, ws_read_limit=ws_read_limit,
timeout = timeout, timeout = timeout,
fp = logfp fp = logfp,
use_http2 = use_http2,
) )
c.connect(connect_to) c.connect(connect_to)
ret = [] ret = []