Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-23 08:11:00 +00:00
Code Issues Projects Releases Wiki Activity

http2: add request-response to pathod

Browse Source
This commit is contained in:
Thomas Kriechbaumer 2015-06-11 16:13:22 +02:00
parent 30fbf57e4b
commit a0d8afd0fc
10 changed files with 297 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
libpathod/language/__init__.py
View File

@ -19,7 +19,7 @@ def expand(msg):
yield msg
def parse_pathod(s):
def parse_pathod(s, use_http2=False):
"""
May raise ParseException
"""
@ -28,12 +28,17 @@ def parse_pathod(s):
except UnicodeError:
raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0)
try:
reqs = pp.Or(
[
if use_http2:
expressions = [
# http2.Frame.expr(),
http2.Response.expr(),
]
else:
expressions = [
websockets.WebsocketFrame.expr(),
http.Response.expr(),
]
).parseString(s, parseAll=True)
reqs = pp.Or(expressions).parseString(s, parseAll=True)
except pp.ParseException as v:
raise exceptions.ParseException(v.msg, v.line, v.col)
return itertools.chain(*[expand(i) for i in reqs])
@ -55,7 +60,6 @@ def parse_pathoc(s, use_http2=False):
websockets.WebsocketClientFrame.expr(),
http.Request.expr(),
]
reqs = pp.OneOrMore(pp.Or(expressions)).parseString(s, parseAll=True)
except pp.ParseException as v:
raise exceptions.ParseException(v.msg, v.line, v.col)

6
libpathod/language/http.py
View File

@ -367,10 +367,6 @@ class Request(_HTTPMessage):
return ":".join([i.spec() for i in self.tokens])
class PathodErrorResponse(Response):
pass
def make_error_response(reason, body=None):
tokens = [
Code("800"),
@ -381,4 +377,4 @@ def make_error_response(reason, body=None):
Reason(base.TokValueLiteral(reason)),
Body(base.TokValueLiteral("pathod error: " + (body or reason))),
]
return PathodErrorResponse(tokens)
return Response(tokens)

115
libpathod/language/http2.py
View File

@ -45,14 +45,20 @@ class Body(base.Value):
class Times(base.Integer):
preamble = "x"
class Code(base.Integer):
pass
class Request(message.Message):
comps = (
Header,
Body,
Times,
)
logattrs = ["method", "path"]
def __init__(self, tokens):
super(Response, self).__init__(tokens)
self.rendered_values = None
@property
def method(self):
@ -87,7 +93,6 @@ class Request(message.Message):
Method.expr(),
base.Sep,
Path.expr(),
base.Sep,
pp.ZeroOrMore(base.Sep + atom)
]
)
@ -95,21 +100,109 @@ class Request(message.Message):
return resp
def resolve(self, settings, msg=None):
tokens = self.tokens[:]
return self.__class__(
[i.resolve(settings, self) for i in tokens]
)
return self
def values(self, settings):
return settings.protocol.create_request(
self.method.string(),
self.path,
self.headers,
self.body)
if self.rendered_values:
return self.rendered_values
else:
headers = self.headers
if headers:
headers = headers.values(settings)
body = self.body
if body:
body = body.string()
self.rendered_values = settings.protocol.create_request(
self.method.string(),
self.path.string(),
headers, # TODO: parse that into a dict?!
body)
return self.rendered_values
def spec(self):
return ":".join([i.spec() for i in self.tokens])
class Response(message.Message):
unique_name = None
comps = (
Header,
Body,
)
def __init__(self, tokens):
super(Response, self).__init__(tokens)
self.rendered_values = None
self.stream_id = 0
@property
def code(self):
return self.tok(Code)
@property
def headers(self):
return self.toks(Header)
@property
def body(self):
return self.tok(Body)
@property
def actions(self):
return []
def resolve(self, settings, msg=None):
return self
@classmethod
def expr(klass):
parts = [i.expr() for i in klass.comps]
atom = pp.MatchFirst(parts)
resp = pp.And(
[
Code.expr(),
pp.ZeroOrMore(base.Sep + atom)
]
)
resp = resp.setParseAction(klass)
return resp
def values(self, settings):
if self.rendered_values:
return self.rendered_values
else:
headers = self.headers
if headers:
headers = headers.values(settings)
body = self.body
if body:
body = body.values(settings)
self.rendered_values = settings.protocol.create_response(
self.code.string(),
self.stream_id,
headers, # TODO: parse that into a dict?!
body)
return self.rendered_values
def spec(self):
return ":".join([i.spec() for i in self.tokens])
def make_error_response(reason, body=None):
raise NotImplementedError
# tokens = [
# Code("800"),
# Header(
# base.TokValueLiteral("Content-Type"),
# base.TokValueLiteral("text/plain")
# ),
# Reason(base.TokValueLiteral(reason)),
# Body(base.TokValueLiteral("pathod error: " + (body or reason))),
# ]
# return Response(tokens)
# class Frame(message.Message):
# pass

8
libpathod/pathoc.py
View File

@ -155,13 +155,14 @@ class Pathoc(tcp.TCPClient):
# SSL
ssl=None,
sni=None,
sslversion=4,
sslversion='SSLv23',
clientcert=None,
ciphers=None,
# HTTP/2
use_http2=False,
http2_skip_connection_preface=False,
http2_framedump = False,
# Websockets
ws_read_limit = None,
@ -199,6 +200,7 @@ class Pathoc(tcp.TCPClient):
self.use_http2 = use_http2
self.http2_skip_connection_preface = http2_skip_connection_preface
self.http2_framedump = http2_framedump
self.ws_read_limit = ws_read_limit
@ -219,7 +221,6 @@ class Pathoc(tcp.TCPClient):
if not OpenSSL._util.lib.Cryptography_HAS_ALPN: # pragma: nocover
print >> sys.stderr, "HTTP/2 requires ALPN support. Please use OpenSSL >= 1.0.2."
print >> sys.stderr, "Pathoc might not be working as expected without ALPN."
self.protocol = http2.HTTP2Protocol(self)
else:
# TODO: create HTTP or Websockets protocol
@ -298,7 +299,7 @@ class Pathoc(tcp.TCPClient):
if self.use_http2:
self.protocol.check_alpn()
if not self.http2_skip_connection_preface:
self.protocol.perform_connection_preface()
self.protocol.perform_client_connection_preface()
if self.timeout:
self.settimeout(self.timeout)
@ -466,6 +467,7 @@ def main(args): # pragma: nocover
ciphers = args.ciphers,
use_http2 = args.use_http2,
http2_skip_connection_preface = args.http2_skip_connection_preface,
http2_framedump = args.http2_framedump,
showreq = args.showreq,
showresp = args.showresp,
explain = args.explain,

15
libpathod/pathoc_cmdline.py
View File

@ -109,12 +109,11 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
help="SSL cipher specification"
)
group.add_argument(
"--sslversion", dest="sslversion", type=int, default=4,
choices=[1, 2, 3, 4],
help="""
Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
to SSLv23.
"""
"--sslversion", dest="sslversion", type=str, default='SSLv23',
choices=utils.SSLVERSIONS.keys(),
help=""""
Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23.
Default to SSLv23."""
)
group = parser.add_argument_group(
@ -156,6 +155,10 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
"-x", dest="hexdump", action="store_true", default=False,
help="Output in hexdump format"
)
group.add_argument(
"--http2-framedump", dest="http2_framedump", action="store_true", default=False,
help="Output all received & sent HTTP/2 frames"
)
args = parser.parse_args(argv[1:])

214
libpathod/pathod.py
View File

@ -7,7 +7,7 @@ import urllib
import re
import time
from netlib import tcp, http, wsgi, certutils, websockets
from netlib import tcp, http, http2, wsgi, certutils, websockets, odict
from . import version, app, language, utils, log
import language.http
@ -40,7 +40,7 @@ class SSLOptions:
sslversion=tcp.SSLv23_METHOD,
ciphers=None,
certs=None,
alpn_select=None,
alpn_select=http2.HTTP2Protocol.ALPN_PROTO_H2,
):
self.confdir = confdir
self.cn = cn
@ -69,32 +69,37 @@ class PathodHandler(tcp.BaseHandler):
wbufsize = 0
sni = None
def __init__(self, connection, address, server, logfp, settings):
self.logfp = logfp
def __init__(self, connection, address, server, logfp, settings, http2_framedump=False):
tcp.BaseHandler.__init__(self, connection, address, server)
self.logfp = logfp
self.settings = copy.copy(settings)
self.protocol = None
self.use_http2 = False
self.http2_framedump = http2_framedump
def handle_sni(self, connection):
def _handle_sni(self, connection):
self.sni = connection.get_servername()
def http_serve_crafted(self, crafted):
"""
This method is HTTP/1 and HTTP/2 capable.
"""
error, crafted = self.server.check_policy(
crafted, self.settings
)
if error:
err = language.http.make_error_response(error)
err = self.make_http_error_response(error)
language.serve(err, self.wfile, self.settings)
return None, dict(
type="error",
msg = error
)
if self.server.explain and not isinstance(
crafted,
language.http.PathodErrorResponse
):
if self.server.explain and not hasattr(crafted, 'is_error_response'):
crafted = crafted.freeze(self.settings)
log.write(self.logfp, ">> Spec: %s" % crafted.spec())
response_log = language.serve(
crafted,
self.wfile,
@ -154,6 +159,8 @@ class PathodHandler(tcp.BaseHandler):
def handle_http_connect(self, connect, lg):
"""
This method is HTTP/1 only.
Handle a CONNECT request.
"""
http.read_headers(self.rfile)
@ -171,7 +178,7 @@ class PathodHandler(tcp.BaseHandler):
self.convert_to_ssl(
cert,
key,
handle_sni=self.handle_sni,
handle_sni=self._handle_sni,
request_client_cert=self.server.ssloptions.request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion,
@ -185,10 +192,12 @@ class PathodHandler(tcp.BaseHandler):
def handle_http_app(self, method, path, headers, content, lg):
"""
This method is HTTP/1 only.
Handle a request to the built-in app.
"""
if self.server.noweb:
crafted = language.http.make_error_response("Access Denied")
crafted = self.make_http_error_response("Access Denied")
language.serve(crafted, self.wfile, self.settings)
return None, dict(
type="error",
@ -209,6 +218,8 @@ class PathodHandler(tcp.BaseHandler):
def handle_http_request(self):
"""
This method is HTTP/1 and HTTP/2 capable.
Returns a (handler, log) tuple.
handler: Handler for the next request, or None to disconnect
@ -217,28 +228,24 @@ class PathodHandler(tcp.BaseHandler):
lr = self.rfile if self.server.logreq else None
lw = self.wfile if self.server.logresp else None
with log.Log(self.logfp, self.server.hexdump, lr, lw) as lg:
line = http.get_request_line(self.rfile)
if not line:
# Normal termination
return None, None
m = utils.MemBool()
if m(http.parse_init_connect(line)):
return self.handle_http_connect(m.v, lg)
elif m(http.parse_init_proxy(line)):
method, _, _, _, path, httpversion = m.v
elif m(http.parse_init_http(line)):
method, path, httpversion = m.v
if self.use_http2:
self.protocol.perform_server_connection_preface()
stream_id, headers, body = self.protocol.read_request()
method = headers[':method']
path = headers[':path']
headers = odict.ODict(headers)
else:
s = "Invalid first line: %s" % repr(line)
lg(s)
return None, dict(type="error", msg=s)
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
lg(s)
return None, dict(type="error", msg=s)
req = self.read_http_request(lg)
if 'next_handle' in req:
return req['next_handle']
if 'errors' in req:
return None, req['errors']
if not 'method' in req or not 'path' in req:
return None, None
method = req['method']
path = req['path']
headers = req['headers']
body = req['body']
clientcert = None
if self.clientcert:
@ -258,7 +265,7 @@ class PathodHandler(tcp.BaseHandler):
path=path,
method=method,
headers=headers.lst,
httpversion=httpversion,
# httpversion=httpversion,
sni=self.sni,
remote_address=self.address(),
clientcert=clientcert,
@ -268,16 +275,6 @@ class PathodHandler(tcp.BaseHandler):
if self.ssl_established:
retlog["cipher"] = self.get_current_cipher()
try:
content = http.read_http_body(
self.rfile, headers, None,
method, None, True
)
except http.HttpError as s:
s = str(s)
lg(s)
return None, dict(type="error", msg=s)
m = utils.MemBool()
websocket_key = websockets.check_client_handshake(headers)
self.settings.websocket_key = websocket_key
@ -288,28 +285,37 @@ class PathodHandler(tcp.BaseHandler):
anchor_gen = language.parse_pathod("ws")
else:
anchor_gen = None
for i in self.server.anchors:
if i[0].match(path):
anchor_gen = i[1]
for regex, spec in self.server.anchors:
if regex.match(path):
anchor_gen = language.parse_pathod(spec, self.use_http2)
break
else:
print(self.server.craftanchor)
if m(path.startswith(self.server.craftanchor)):
spec = urllib.unquote(path)[len(self.server.craftanchor):]
if spec:
try:
anchor_gen = language.parse_pathod(spec)
anchor_gen = language.parse_pathod(spec, self.use_http2)
except language.ParseException as v:
lg("Parse error: %s" % v.msg)
anchor_gen = iter([language.http.make_error_response(
anchor_gen = iter([self.make_http_error_response(
"Parse Error",
"Error parsing response spec: %s\n" % (
v.msg + v.marked()
)
)])
else:
if self.use_http2:
raise NotImplementedError(\
"HTTP/2 only supports request/response with the craft anchor point.")
if anchor_gen:
spec = anchor_gen.next()
if self.use_http2 and isinstance(spec, language.http2.Response):
spec.stream_id = stream_id
lg("crafting spec: %s" % spec)
nexthandler, retlog["response"] = self.http_serve_crafted(
spec
@ -319,28 +325,77 @@ class PathodHandler(tcp.BaseHandler):
else:
return nexthandler, retlog
else:
return self.handle_http_app(method, path, headers, content, lg)
return self.handle_http_app(method, path, headers, body, lg)
def addlog(self, log):
# FIXME: The bytes in the log should not be escaped. We do this at the
# moment because JSON encoding can't handle binary data, and I don't
# want to base64 everything.
if self.server.logreq:
bytes = self.rfile.get_log().encode("string_escape")
log["request_bytes"] = bytes
if self.server.logresp:
bytes = self.wfile.get_log().encode("string_escape")
log["response_bytes"] = bytes
self.server.add_log(log)
def read_http_request(self, lg):
"""
This method is HTTP/1 only.
"""
line = http.get_request_line(self.rfile)
if not line:
# Normal termination
return dict()
m = utils.MemBool()
if m(http.parse_init_connect(line)):
return dict(next_handle=self.handle_http_connect(m.v, lg))
elif m(http.parse_init_proxy(line)):
method, _, _, _, path, httpversion = m.v
elif m(http.parse_init_http(line)):
method, path, httpversion = m.v
else:
s = "Invalid first line: %s" % repr(line)
lg(s)
return dict(errors=dict(type="error", msg=s))
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
lg(s)
return dict(errors=dict(type="error", msg=s))
try:
body = http.read_http_body(
self.rfile,
headers,
None,
method,
None,
True,
)
except http.HttpError as s:
s = str(s)
lg(s)
return dict(errors=dict(type="error", msg=s))
return dict(
method=method,
path=path,
headers=headers,
body=body)
def make_http_error_response(self, reason, body=None):
"""
This method is HTTP/1 and HTTP/2 capable.
"""
if self.use_http2:
resp = language.http2.make_error_response(reason, body)
else:
resp = language.http.make_error_response(reason, body)
resp.is_error_response = True
return resp
def handle(self):
self.settimeout(self.server.timeout)
if self.server.ssl:
try:
cert, key, _ = self.server.ssloptions.get_cert(None)
self.convert_to_ssl(
cert,
key,
handle_sni=self.handle_sni,
dhparams=self.server.ssloptions.certstore.dhparams,
handle_sni=self._handle_sni,
request_client_cert=self.server.ssloptions.request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion,
@ -356,8 +411,20 @@ class PathodHandler(tcp.BaseHandler):
)
log.write(self.logfp, s)
return
self.settimeout(self.server.timeout)
alp = self.get_alpn_proto_negotiated()
if alp == http2.HTTP2Protocol.ALPN_PROTO_H2:
self.protocol = http2.HTTP2Protocol(self, is_server=True, dump_frames=self.http2_framedump)
self.use_http2 = True
# if not self.protocol:
# # TODO: create HTTP or Websockets protocol
# self.protocol = None
self.settings.protocol = self.protocol
handler = self.handle_http_request
while not self.finished:
handler, l = handler()
if l:
@ -365,6 +432,18 @@ class PathodHandler(tcp.BaseHandler):
if not handler:
return
def addlog(self, log):
# FIXME: The bytes in the log should not be escaped. We do this at the
# moment because JSON encoding can't handle binary data, and I don't
# want to base64 everything.
if self.server.logreq:
bytes = self.rfile.get_log().encode("string_escape")
log["request_bytes"] = bytes
if self.server.logresp:
bytes = self.wfile.get_log().encode("string_escape")
log["response_bytes"] = bytes
self.server.add_log(log)
class Pathod(tcp.TCPServer):
LOGBUF = 500
@ -387,6 +466,7 @@ class Pathod(tcp.TCPServer):
logresp=False,
explain=False,
hexdump=False,
http2_framedump=False,
webdebug=False,
logfp=sys.stdout,
):
@ -414,6 +494,7 @@ class Pathod(tcp.TCPServer):
self.noapi, self.nohang = noapi, nohang
self.timeout, self.logreq = timeout, logreq
self.logresp, self.hexdump = logresp, hexdump
self.http2_framedump = http2_framedump
self.explain = explain
self.logfp = logfp
@ -451,7 +532,8 @@ class Pathod(tcp.TCPServer):
client_address,
self,
self.logfp,
self.settings
self.settings,
self.http2_framedump,
)
try:
h.handle()
@ -508,7 +590,6 @@ def main(args): # pragma: nocover
sslversion = utils.SSLVERSIONS[args.sslversion],
certs = args.ssl_certs,
sans = args.sans,
alpn_select = args.alpn_select,
)
root = logging.getLogger()
@ -548,6 +629,7 @@ def main(args): # pragma: nocover
logreq = args.logreq,
logresp = args.logresp,
hexdump = args.hexdump,
http2_framedump = args.http2_framedump,
explain = args.explain,
webdebug = args.webdebug
)

15
libpathod/pathod_cmdline.py
View File

@ -139,10 +139,11 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr):
"""
)
group.add_argument(
"--sslversion", dest="sslversion", type=int, default=4,
choices=[1, 2, 3, 4],
help=""""Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
to SSLv23."""
"--sslversion", dest="sslversion", type=str, default='SSLv23',
choices=utils.SSLVERSIONS.keys(),
help=""""
Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23.
Default to SSLv23."""
)
group = parser.add_argument_group(
@ -172,6 +173,12 @@ def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr):
"-x", dest="hexdump", action="store_true", default=False,
help="Log request/response in hexdump format"
)
group.add_argument(
"--http2-framedump", dest="http2_framedump", action="store_true", default=False,
help="Output all received & sent HTTP/2 frames"
)
args = parser.parse_args(argv[1:])
certs = []

10
libpathod/utils.py
View File

@ -3,10 +3,12 @@ import sys
from netlib import tcp
SSLVERSIONS = {
1: tcp.TLSv1_METHOD,
2: tcp.SSLv2_METHOD,
3: tcp.SSLv3_METHOD,
4: tcp.SSLv23_METHOD,
'TLSv1.2': tcp.TLSv1_2_METHOD,
'TLSv1.1': tcp.TLSv1_1_METHOD,
'TLSv1': tcp.TLSv1_METHOD,
'SSLv3': tcp.SSLv3_METHOD,
'SSLv2': tcp.SSLv2_METHOD,
'SSLv23': tcp.SSLv23_METHOD,
}
SIZE_UNITS = dict(

3
test/test_language_http2.py
View File

@ -1,5 +1,6 @@
import cStringIO
from netlib import tcp
from libpathod import language
from libpathod.language import http2, base
import netlib
@ -64,7 +65,7 @@ class TestRequest:
s,
language.Settings(
request_host = "foo.com",
protocol = netlib.http2.HTTP2Protocol(None)
protocol = netlib.http2.HTTP2Protocol(tcp.TCPClient(('localhost', 1234)))
)
)

7
test/test_pathoc.py
View File

@ -230,9 +230,6 @@ class TestDaemon(_TestDaemon):
class TestDaemonHTTP2(_TestDaemon):
ssl = True
ssloptions = pathod.SSLOptions(
alpn_select = http2.HTTP2Protocol.ALPN_PROTO_H2,
)
def test_http2(self):
c = pathoc.Pathoc(
@ -270,5 +267,5 @@ class TestDaemonHTTP2(_TestDaemon):
use_http2 = True,
)
c.connect()
resp = c.request("get:/api/info")
assert tuple(json.loads(resp.content)["version"]) == version.IVERSION
resp = c.request("get:/p/200")
assert resp.status_code == "200"