Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-23 00:01:36 +00:00
Code Issues Projects Releases Wiki Activity

Basic HTTP/1.1 Support

Browse Source 
Adds support for chunked transfer encoding, and a couple other minor
protocol corrections.

Improve HTTP support

- Support intercepted requests with Host header
- Support HEAD requests proper
- Support any HTTP method including extensions, not just a couple known ones

Support expect: 100-continue and 100 Continue messages

Persistent client connections

Generalize ServerConnection a bit in preparation for keep-alive support

Correct HTTP status codes on errors forwarding the request
This commit is contained in:
Henrik Nordstrom 2010-11-12 16:01:17 +01:00 committed by Henrik Nordstrom
parent 0613321aef
commit 4bae297fbb
2 changed files with 175 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libmproxy/console.py
View File

@ -447,7 +447,7 @@ class ConnectionView(WWrap):
self.master.prompt_edit("Message", conn.msg, self.set_resp_msg) self.master.prompt_edit("Message", conn.msg, self.set_resp_msg)
elif part == "r" and self.state.view_flow_mode == VIEW_FLOW_REQUEST: elif part == "r" and self.state.view_flow_mode == VIEW_FLOW_REQUEST:
if not conn.acked: if not conn.acked:
response = proxy.Response(conn, "200", "HTTP/1.1", "OK", utils.Headers(), "") response = proxy.Response(conn, "200", "OK", utils.Headers(), "")
conn.ack(response) conn.ack(response)
self.view_response() self.view_response()
self.master.refresh_connection(self.flow) self.master.refresh_connection(self.flow)

227
libmproxy/proxy.py
View File

@ -26,6 +26,45 @@ class Config:
self.pemfile = pemfile self.pemfile = pemfile
def read_chunked(fp):
content = ""
while 1:
line = fp.readline()
if line == "":
raise IOError("Connection closed")
if line == '\r\n' or line == '\n':
continue
length = int(line,16)
if not length:
break
content += fp.read(length)
line = fp.readline()
if line != '\r\n':
raise IOError("Malformed chunked body")
while 1:
line = fp.readline()
if line == "":
raise IOError("Connection closed")
if line == '\r\n' or line == '\n':
break
return content
def read_http_body(rfile, connection, headers, all):
if headers.has_key('transfer-encoding'):
if not ",".join(headers["transfer-encoding"]) == "chunked":
raise IOError('Invalid transfer-encoding')
content = read_chunked(rfile)
elif headers.has_key("content-length"):
content = rfile.read(int(headers["content-length"][0]))
elif all:
content = rfile.read()
connection.close = True
else:
content = None
return content
def parse_url(url): def parse_url(url):
""" """
Returns a (scheme, host, port, path) tuple, or None on error. Returns a (scheme, host, port, path) tuple, or None on error.
@ -48,7 +87,7 @@ def parse_url(url):
return scheme, host, port, path return scheme, host, port, path
def parse_proxy_request(request): def parse_request_line(request):
""" """
Parse a proxy request line. Return (method, scheme, host, port, path). Parse a proxy request line. Return (method, scheme, host, port, path).
Raise ProxyError on error. Raise ProxyError on error.
@ -57,31 +96,40 @@ def parse_proxy_request(request):
method, url, protocol = string.split(request) method, url, protocol = string.split(request)
except ValueError: except ValueError:
raise ProxyError(400, "Can't parse request") raise ProxyError(400, "Can't parse request")
if method in ['GET', 'HEAD', 'POST']: if method == 'CONNECT':
if url.startswith("/"): scheme = None
path = None
try:
host, port = url.split(":")
except ValueError:
raise ProxyError(400, "Can't parse request")
port = int(port)
else:
if url.startswith("/") or url == "*":
scheme, port, host, path = None, None, None, url scheme, port, host, path = None, None, None, url
else: else:
parts = parse_url(url) parts = parse_url(url)
if not parts: if not parts:
raise ProxyError(400, "Invalid url: %s"%url) raise ProxyError(400, "Invalid url: %s"%url)
scheme, host, port, path = parts scheme, host, port, path = parts
elif method == 'CONNECT': if not protocol.startswith("HTTP/"):
scheme = None raise ProxyError(400, "Unsupported protocol")
path = None major,minor = protocol.split('/')[1].split('.')
host, port = url.split(":") major = int(major)
port = int(port) minor = int(minor)
else: if major != 1:
raise ProxyError(501, "Unknown request method: %s" % method) raise ProxyError(400, "Unsupported protocol")
return method, scheme, host, port, path return method, scheme, host, port, path, minor
class Request(controller.Msg): class Request(controller.Msg):
FMT = '%s %s HTTP/1.0\r\n%s\r\n%s' FMT = '%s %s HTTP/1.1\r\n%s\r\n%s'
def __init__(self, client_conn, host, port, scheme, method, path, headers, content, timestamp=None): def __init__(self, client_conn, host, port, scheme, method, path, headers, content, timestamp=None):
self.client_conn = client_conn self.client_conn = client_conn
self.host, self.port, self.scheme = host, port, scheme self.host, self.port, self.scheme = host, port, scheme
self.method, self.path, self.headers, self.content = method, path, headers, content self.method, self.path, self.headers, self.content = method, path, headers, content
self.timestamp = timestamp or time.time() self.timestamp = timestamp or time.time()
self.close = False
controller.Msg.__init__(self) controller.Msg.__init__(self)
def get_state(self): def get_state(self):
@ -118,12 +166,15 @@ class Request(controller.Msg):
c.headers = self.headers.copy() c.headers = self.headers.copy()
return c return c
def url(self): def hostport(self):
if (self.port, self.scheme) in [(80, "http"), (443, "https")]: if (self.port, self.scheme) in [(80, "http"), (443, "https")]:
host = self.host host = self.host
else: else:
host = "%s:%s"%(self.host, self.port) host = "%s:%s"%(self.host, self.port)
return "%s://%s%s"%(self.scheme, host, self.path) return host
def url(self):
return "%s://%s%s"%(self.scheme, self.hostport(), self.path)
def set_url(self, url): def set_url(self, url):
parts = parse_url(url) parts = parse_url(url)
@ -148,16 +199,26 @@ class Request(controller.Msg):
utils.try_del(headers, 'proxy-connection') utils.try_del(headers, 'proxy-connection')
utils.try_del(headers, 'keep-alive') utils.try_del(headers, 'keep-alive')
utils.try_del(headers, 'connection') utils.try_del(headers, 'connection')
utils.try_del(headers, 'content-length')
utils.try_del(headers, 'transfer-encoding')
if not headers.has_key('host'):
headers["host"] = [self.hostport()]
content = self.content
if content is not None:
headers["content-length"] = [str(len(content))]
else:
content = ""
if self.close:
headers["connection"] = ["close"] headers["connection"] = ["close"]
data = (self.method, self.path, str(headers), self.content) data = (self.method, self.path, str(headers), content)
return self.FMT%data return self.FMT%data
class Response(controller.Msg): class Response(controller.Msg):
FMT = '%s\r\n%s\r\n%s' FMT = '%s\r\n%s\r\n%s'
def __init__(self, request, code, proto, msg, headers, content, timestamp=None): def __init__(self, request, code, msg, headers, content, timestamp=None):
self.request = request self.request = request
self.code, self.proto, self.msg = code, proto, msg self.code, self.msg = code, msg
self.headers, self.content = headers, content self.headers, self.content = headers, content
self.timestamp = timestamp or time.time() self.timestamp = timestamp or time.time()
controller.Msg.__init__(self) controller.Msg.__init__(self)
@ -196,7 +257,7 @@ class Response(controller.Msg):
return True return True
def short(self): def short(self):
return "%s %s"%(self.code, self.proto) return "%s %s"%(self.code, self.msg)
def assemble(self): def assemble(self):
""" """
@ -208,9 +269,16 @@ class Response(controller.Msg):
utils.try_del(headers, 'proxy-connection') utils.try_del(headers, 'proxy-connection')
utils.try_del(headers, 'connection') utils.try_del(headers, 'connection')
utils.try_del(headers, 'keep-alive') utils.try_del(headers, 'keep-alive')
utils.try_del(headers, 'transfer-encoding')
content = self.content
if content is not None:
headers["content-length"] = [str(len(content))]
else:
content = ""
if self.request.client_conn.close:
headers["connection"] = ["close"] headers["connection"] = ["close"]
proto = "%s %s %s"%(self.proto, self.code, self.msg) proto = "HTTP/1.1 %s %s"%(self.code, self.msg)
data = (proto, str(headers), self.content) data = (proto, str(headers), content)
return self.FMT%data return self.FMT%data
@ -221,6 +289,7 @@ class ClientConnection(controller.Msg):
been replayed from within mitmproxy. been replayed from within mitmproxy.
""" """
self.address = address self.address = address
self.close = False
controller.Msg.__init__(self) controller.Msg.__init__(self)
def get_state(self): def get_state(self):
@ -306,46 +375,54 @@ class FileLike:
class ServerConnection: class ServerConnection:
def __init__(self, request): def __init__(self, request):
self.request = request self.host = request.host
self.port = request.port
self.scheme = request.scheme
self.close = False
self.server, self.rfile, self.wfile = None, None, None self.server, self.rfile, self.wfile = None, None, None
self.connect() self.connect()
self.send_request()
def connect(self): def connect(self):
try: try:
addr = socket.gethostbyname(self.request.host) addr = socket.gethostbyname(self.host)
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if self.request.scheme == "https": if self.scheme == "https":
server = ssl.wrap_socket(server) server = ssl.wrap_socket(server)
server.connect((addr, self.request.port)) server.connect((addr, self.port))
except socket.error, err: except socket.error, err:
raise ProxyError(200, 'Error connecting to "%s": %s' % (self.request.host, err)) raise ProxyError(504, 'Error connecting to "%s": %s' % (self.host, err))
self.server = server self.server = server
self.rfile, self.wfile = server.makefile('rb'), server.makefile('wb') self.rfile, self.wfile = server.makefile('rb'), server.makefile('wb')
def send_request(self): def send_request(self, request):
self.request = request
request.close = self.close
try: try:
self.wfile.write(self.request.assemble()) self.wfile.write(request.assemble())
self.wfile.flush() self.wfile.flush()
except socket.error, err: except socket.error, err:
raise ProxyError(500, 'Error sending data to "%s": %s' % (request.host, err)) raise ProxyError(504, 'Error sending data to "%s": %s' % (request.host, err))
def read_response(self): def read_response(self):
proto = self.rfile.readline() line = self.rfile.readline()
if not proto: if line == "\r\n" or line == "\n": # Possible leftover from previous message
raise ProxyError(200, "Blank server response.") line = self.rfile.readline()
parts = proto.strip().split(" ", 2) if not line:
raise ProxyError(502, "Blank server response.")
parts = line.strip().split(" ", 2)
if not len(parts) == 3: if not len(parts) == 3:
raise ProxyError(200, "Invalid server response: %s."%proto) raise ProxyError(502, "Invalid server response: %s."%line)
proto, code, msg = parts proto, code, msg = parts
code = int(code) code = int(code)
headers = utils.Headers() headers = utils.Headers()
headers.read(self.rfile) headers.read(self.rfile)
if headers.has_key("content-length"): if code >= 100 and code <= 199:
content = self.rfile.read(int(headers["content-length"][0])) return self.read_response()
if self.request.method == "HEAD" or code == 204 or code == 304:
content = None
else: else:
content = self.rfile.read() content = read_http_body(self.rfile, self, headers, True)
return Response(self.request, code, proto, msg, headers, content) return Response(self.request, code, msg, headers, content)
def terminate(self): def terminate(self):
try: try:
@ -362,14 +439,22 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
SocketServer.StreamRequestHandler.__init__(self, request, client_address, server) SocketServer.StreamRequestHandler.__init__(self, request, client_address, server)
def handle(self): def handle(self):
server = None
cc = ClientConnection(self.client_address) cc = ClientConnection(self.client_address)
cc.send(self.mqueue) cc.send(self.mqueue)
while not cc.close:
self.handle_request(cc)
self.finish()
def handle_request(self, cc):
server = None
try: try:
request = self.read_request(cc) request = self.read_request(cc)
if request is None:
cc.close = True
return
request = request.send(self.mqueue) request = request.send(self.mqueue)
if request is None: if request is None:
self.finish() cc.close = True
return return
if request.is_response(): if request.is_response():
response = request response = request
@ -377,12 +462,13 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
response = response.send(self.mqueue) response = response.send(self.mqueue)
else: else:
server = ServerConnection(request) server = ServerConnection(request)
server.send_request(request)
response = server.read_response() response = server.read_response()
response = response.send(self.mqueue) response = response.send(self.mqueue)
if response is None: if response is None:
server.terminate() server.terminate()
if response is None: if response is None:
self.finish() cc.close = True
return return
self.send_response(response) self.send_response(response)
except IOError: except IOError:
@ -390,22 +476,24 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
except ProxyError, e: except ProxyError, e:
err = Error(cc, e.msg) err = Error(cc, e.msg)
err.send(self.mqueue) err.send(self.mqueue)
cc.close = True
self.send_error(e.code, e.msg) self.send_error(e.code, e.msg)
if server: if server:
server.terminate() server.terminate()
self.finish()
def read_request(self, client_conn): def read_request(self, client_conn):
request = self.rfile.readline() line = self.rfile.readline()
method, scheme, host, port, path = parse_proxy_request(request) if line == "\r\n" or line == "\n": # Possible leftover from previous message
if not host: line = self.rfile.readline()
raise ProxyError(200, 'Invalid request: %s'%request) if line == "":
return None
method, scheme, host, port, path, httpminor = parse_request_line(line)
if method == "CONNECT": if method == "CONNECT":
# Discard additional headers sent to the proxy. Should I expose # Discard additional headers sent to the proxy. Should I expose
# these to users? # these to users?
while 1: while 1:
d = self.rfile.readline() d = self.rfile.readline()
if not d.strip(): if d == '\r\n' or d == '\n':
break break
self.wfile.write( self.wfile.write(
'HTTP/1.1 200 Connection established\r\n' + 'HTTP/1.1 200 Connection established\r\n' +
@ -423,16 +511,44 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
) )
self.rfile = FileLike(self.connection) self.rfile = FileLike(self.connection)
self.wfile = FileLike(self.connection) self.wfile = FileLike(self.connection)
method, _, _, _, path = parse_proxy_request(self.rfile.readline()) method, scheme, host, port, path, httpminor = parse_request_line(self.rfile.readline())
if scheme is None:
scheme = "https" scheme = "https"
headers = utils.Headers() headers = utils.Headers()
headers.read(self.rfile) headers.read(self.rfile)
if method == 'POST' and not headers.has_key('content-length'): if host is None and headers.has_key("host"):
raise ProxyError(400, "Missing Content-Length for POST method") netloc = headers["host"][0]
if headers.has_key("content-length") and int(headers["content-length"][0]): if ':' in netloc:
content = self.rfile.read(int(headers["content-length"][0])) host, port = string.split(netloc, ':')
port = int(port)
else: else:
content = "" host = netloc
if scheme == "https":
port = 443
else:
port = 80
port = int(port)
if host is None:
raise ProxyError(400, 'Invalid request: %s'%request)
if headers.has_key('expect'):
expect = ",".join(headers['expect'])
if expect == "100-continue" and httpminor >= 1:
self.wfile.write('HTTP/1.1 100 Continue\r\n')
self.wfile.write('Proxy-agent: %s\r\n'%NAME)
self.wfile.write('\r\n')
del headers['expect']
else:
raise ProxyError(417, 'Unmet expect: %s'%expect)
if httpminor == 0:
client_conn.close = True
if headers.has_key('connection'):
for value in ",".join(headers['connection']).split(","):
value = value.strip()
if value == "close":
client_conn.close = True
if value == "keep-alive":
client_conn.close = False
content = read_http_body(self.rfile, client_conn, headers, False)
return Request(client_conn, host, port, scheme, method, path, headers, content) return Request(client_conn, host, port, scheme, method, path, headers, content)
def send_response(self, response): def send_response(self, response):
@ -455,15 +571,14 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
try: try:
import BaseHTTPServer import BaseHTTPServer
response = BaseHTTPServer.BaseHTTPRequestHandler.responses[code][0] response = BaseHTTPServer.BaseHTTPRequestHandler.responses[code][0]
self.wfile.write("HTTP/1.0 %s %s\r\n" % (code, response)) self.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response))
self.wfile.write("Server: %s\r\n"%NAME) self.wfile.write("Server: %s\r\n"%NAME)
self.wfile.write("Connection: close\r\n")
self.wfile.write("Content-type: text/html\r\n") self.wfile.write("Content-type: text/html\r\n")
self.wfile.write("\r\n") self.wfile.write("\r\n")
self.wfile.write('<html><head>\n<title>%d %s</title>\n</head>\n' self.wfile.write('<html><head>\n<title>%d %s</title>\n</head>\n'
'<body>\n%s\n</body>\n</html>' % (code, response, body)) '<body>\n%s\n</body>\n</html>' % (code, response, body))
self.wfile.flush() self.wfile.flush()
self.wfile.close()
self.rfile.close()
except IOError: except IOError:
pass pass