# Copyright (C) 2012 Aldo Cortesi # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ A simple proxy server implementation, which always reads all of a server response into memory, performs some transformation, and then writes it back to the client. """ import sys, os, string, socket, time import shutil, tempfile, threading import optparse, SocketServer, ssl import utils, flow NAME = "mitmproxy" class ProxyError(Exception): def __init__(self, code, msg): self.code, self.msg = code, msg def __str__(self): return "ProxyError(%s, %s)"%(self.code, self.msg) class ProxyConfig: def __init__(self, certfile = None, ciphers = None, cacert = None, cert_wait_time=0, upstream_cert=False, body_size_limit = None, reverse_proxy=None): self.certfile = certfile self.ciphers = ciphers self.cacert = cacert self.certdir = None self.cert_wait_time = cert_wait_time self.upstream_cert = upstream_cert self.body_size_limit = body_size_limit self.reverse_proxy = reverse_proxy def read_headers(fp): """ Read a set of headers from a file pointer. Stop once a blank line is reached. Return a ODict object. """ ret = [] name = '' while 1: line = fp.readline() if not line or line == '\r\n' or line == '\n': break if line[0] in ' \t': # continued header ret[-1][1] = ret[-1][1] + '\r\n ' + line.strip() else: i = line.find(':') # We're being liberal in what we accept, here. if i > 0: name = line[:i] value = line[i+1:].strip() ret.append([name, value]) return flow.ODictCaseless(ret) def read_chunked(fp, limit): content = "" total = 0 while 1: line = fp.readline(128) if line == "": raise IOError("Connection closed") if line == '\r\n' or line == '\n': continue try: length = int(line,16) except ValueError: # FIXME: Not strictly correct - this could be from the server, in which # case we should send a 502. raise ProxyError(400, "Invalid chunked encoding length: %s"%line) if not length: break total += length if limit is not None and total > limit: msg = "HTTP Body too large."\ " Limit is %s, chunked content length was at least %s"%(limit, total) raise ProxyError(509, msg) content += fp.read(length) line = fp.readline(5) if line != '\r\n': raise IOError("Malformed chunked body") while 1: line = fp.readline() if line == "": raise IOError("Connection closed") if line == '\r\n' or line == '\n': break return content def read_http_body(rfile, connection, headers, all, limit): if 'transfer-encoding' in headers: if not ",".join(headers["transfer-encoding"]) == "chunked": raise IOError('Invalid transfer-encoding') content = read_chunked(rfile, limit) elif "content-length" in headers: try: l = int(headers["content-length"][0]) except ValueError: # FIXME: Not strictly correct - this could be from the server, in which # case we should send a 502. raise ProxyError(400, "Invalid content-length header: %s"%headers["content-length"]) if limit is not None and l > limit: raise ProxyError(509, "HTTP Body too large. Limit is %s, content-length was %s"%(limit, l)) content = rfile.read(l) elif all: content = rfile.read(limit if limit else None) connection.close = True else: content = "" return content def parse_request_line(request): """ Parse a proxy request line. Return (method, scheme, host, port, path, minor). Raise ProxyError on error. """ try: method, url, protocol = string.split(request) except ValueError: raise ProxyError(400, "Can't parse request") if method == 'CONNECT': scheme = None path = None try: host, port = url.split(":") except ValueError: raise ProxyError(400, "Can't parse request") port = int(port) else: if url.startswith("/") or url == "*": scheme, port, host, path = None, None, None, url else: parts = utils.parse_url(url) if not parts: raise ProxyError(400, "Invalid url: %s"%url) scheme, host, port, path = parts if not protocol.startswith("HTTP/"): raise ProxyError(400, "Unsupported protocol") major,minor = protocol.split('/')[1].split('.') major = int(major) minor = int(minor) if major != 1: raise ProxyError(400, "Unsupported protocol") return method, scheme, host, port, path, minor class FileLike: def __init__(self, o): self.o = o def __getattr__(self, attr): return getattr(self.o, attr) def flush(self): pass def read(self, length): result = '' while len(result) < length: try: data = self.o.read(length) except AttributeError: break if not data: break result += data return result def readline(self): result = '' while True: ch = self.read(1) if not ch: break else: result += ch if ch == '\n': break return result #begin nocover class RequestReplayThread(threading.Thread): def __init__(self, config, flow, masterq): self.config, self.flow, self.masterq = config, flow, masterq threading.Thread.__init__(self) def run(self): try: server = ServerConnection(self.config, self.flow.request) server.send() response = server.read_response() response._send(self.masterq) except ProxyError, v: err = flow.Error(self.flow.request, v.msg) err._send(self.masterq) class ServerConnection: def __init__(self, config, request): self.config, self.request = config, request if config.reverse_proxy: self.scheme, self.host, self.port = config.reverse_proxy else: self.host = request.host self.port = request.port self.scheme = request.scheme self.close = False self.server, self.rfile, self.wfile = None, None, None self.connect() def connect(self): try: addr = socket.gethostbyname(self.host) server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if self.scheme == "https": server = ssl.wrap_socket(server) server.connect((addr, self.port)) except socket.error, err: raise ProxyError(502, 'Error connecting to "%s": %s' % (self.host, err)) self.server = server self.rfile, self.wfile = server.makefile('rb'), server.makefile('wb') def send(self): self.request.close = self.close try: self.wfile.write(self.request._assemble()) self.wfile.flush() except socket.error, err: raise ProxyError(502, 'Error sending data to "%s": %s' % (self.request.host, err)) def read_response(self): line = self.rfile.readline() if line == "\r\n" or line == "\n": # Possible leftover from previous message line = self.rfile.readline() if not line: raise ProxyError(502, "Blank server response.") parts = line.strip().split(" ", 2) if len(parts) == 2: # handle missing message gracefully parts.append("") if not len(parts) == 3: raise ProxyError(502, "Invalid server response: %s."%line) proto, code, msg = parts try: code = int(code) except ValueError: raise ProxyError(502, "Invalid server response: %s."%line) headers = read_headers(self.rfile) if code >= 100 and code <= 199: return self.read_response() if self.request.method == "HEAD" or code == 204 or code == 304: content = "" else: content = read_http_body(self.rfile, self, headers, True, self.config.body_size_limit) return flow.Response(self.request, code, msg, headers, content) def terminate(self): try: if not self.wfile.closed: self.wfile.flush() self.server.close() except IOError: pass class ProxyHandler(SocketServer.StreamRequestHandler): def __init__(self, config, request, client_address, server, q): self.config = config self.mqueue = q SocketServer.StreamRequestHandler.__init__(self, request, client_address, server) def handle(self): cc = flow.ClientConnect(self.client_address) cc._send(self.mqueue) while not cc.close: self.handle_request(cc) cd = flow.ClientDisconnect(cc) cd._send(self.mqueue) self.finish() def handle_request(self, cc): server, request, err = None, None, None try: try: request = self.read_request(cc) except IOError, v: raise IOError, "Reading request: %s"%v if request is None: cc.close = True return cc.requestcount += 1 request = request._send(self.mqueue) if request is None: cc.close = True return if isinstance(request, flow.Response): response = request request = False response = response._send(self.mqueue) else: server = ServerConnection(self.config, request) server.send() try: response = server.read_response() except IOError, v: raise IOError, "Reading response: %s"%v response = response._send(self.mqueue) if response is None: server.terminate() if response is None: cc.close = True return self.send_response(response) except IOError, v: cc.connection_error = v cc.close = True except ProxyError, e: cc.close = True cc.connection_error = "%s: %s"%(e.code, e.msg) if request: err = flow.Error(request, e.msg) err._send(self.mqueue) self.send_error(e.code, e.msg) if server: server.terminate() def find_cert(self, host, port): if self.config.certfile: return self.config.certfile else: sans = [] if self.config.upstream_cert: host, sans = utils.get_remote_cn(host, port) ret = utils.dummy_cert(self.config.certdir, self.config.cacert, host, sans) time.sleep(self.config.cert_wait_time) if not ret: raise ProxyError(502, "mitmproxy: Unable to generate dummy cert.") return ret def read_request(self, client_conn): line = self.rfile.readline() if line == "\r\n" or line == "\n": # Possible leftover from previous message line = self.rfile.readline() if line == "": return None method, scheme, host, port, path, httpminor = parse_request_line(line) if method == "CONNECT": # Discard additional headers sent to the proxy. Should I expose # these to users? while 1: d = self.rfile.readline() if d == '\r\n' or d == '\n': break self.wfile.write( 'HTTP/1.1 200 Connection established\r\n' + ('Proxy-agent: %s\r\n'%NAME) + '\r\n' ) self.wfile.flush() kwargs = dict( certfile = self.find_cert(host, port), keyfile = self.config.certfile or self.config.cacert, server_side = True, ssl_version = ssl.PROTOCOL_SSLv23, do_handshake_on_connect = True, ) if sys.version_info[1] > 6: kwargs["ciphers"] = self.config.ciphers self.connection = ssl.wrap_socket(self.connection, **kwargs) self.rfile = FileLike(self.connection) self.wfile = FileLike(self.connection) method, scheme, host, port, path, httpminor = parse_request_line(self.rfile.readline()) if scheme is None: scheme = "https" headers = read_headers(self.rfile) if host is None and "host" in headers: netloc = headers["host"][0] if ':' in netloc: host, port = string.split(netloc, ':') port = int(port) else: host = netloc if scheme == "https": port = 443 else: port = 80 port = int(port) if host is None: if self.config.reverse_proxy: scheme, host, port = self.config.reverse_proxy else: # FIXME: We only specify the first part of the invalid request in this error. # We should gather up everything read from the socket, and specify it all. raise ProxyError(400, 'Invalid request: %s'%line) if "expect" in headers: expect = ",".join(headers['expect']) if expect == "100-continue" and httpminor >= 1: self.wfile.write('HTTP/1.1 100 Continue\r\n') self.wfile.write('Proxy-agent: %s\r\n'%NAME) self.wfile.write('\r\n') del headers['expect'] else: raise ProxyError(417, 'Unmet expect: %s'%expect) if httpminor == 0: client_conn.close = True if "connection" in headers: for value in ",".join(headers['connection']).split(","): value = value.strip() if value == "close": client_conn.close = True if value == "keep-alive": client_conn.close = False content = read_http_body(self.rfile, client_conn, headers, False, self.config.body_size_limit) return flow.Request(client_conn, host, port, scheme, method, path, headers, content) def send_response(self, response): self.wfile.write(response._assemble()) self.wfile.flush() def terminate(self, connection, wfile, rfile): self.request.close() try: if not getattr(wfile, "closed", False): wfile.flush() connection.close() except IOError: pass def finish(self): self.terminate(self.connection, self.wfile, self.rfile) def send_error(self, code, body): try: import BaseHTTPServer response = BaseHTTPServer.BaseHTTPRequestHandler.responses[code][0] self.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response)) self.wfile.write("Server: %s\r\n"%NAME) self.wfile.write("Connection: close\r\n") self.wfile.write("Content-type: text/html\r\n") self.wfile.write("\r\n") self.wfile.write('\n%d %s\n\n' '\n%s\n\n' % (code, response, body)) self.wfile.flush() except: pass class ProxyServerError(Exception): pass ServerBase = SocketServer.ThreadingTCPServer ServerBase.daemon_threads = True # Terminate workers when main thread terminates class ProxyServer(ServerBase): request_queue_size = 20 allow_reuse_address = True def __init__(self, config, port, address=''): """ Raises ProxyServerError if there's a startup problem. """ self.config, self.port, self.address = config, port, address try: ServerBase.__init__(self, (address, port), ProxyHandler) except socket.error, v: raise ProxyServerError('Error starting proxy server: ' + v.strerror) self.masterq = None self.certdir = tempfile.mkdtemp(prefix="mitmproxy") config.certdir = self.certdir def set_mqueue(self, q): self.masterq = q def finish_request(self, request, client_address): self.RequestHandlerClass(self.config, request, client_address, self, self.masterq) def shutdown(self): ServerBase.shutdown(self) try: shutil.rmtree(self.certdir) except OSError: pass # Command-line utils def certificate_option_group(parser): group = optparse.OptionGroup(parser, "SSL") group.add_option( "--cert", action="store", type = "str", dest="cert", default=None, help = "User-created SSL certificate file." ) group.add_option( "--ciphers", action="store", type = "str", dest="ciphers", default=None, help = "SSL ciphers." ) parser.add_option_group(group) def process_proxy_options(parser, options): if options.cert: options.cert = os.path.expanduser(options.cert) if not os.path.exists(options.cert): parser.error("Manually created certificate does not exist: %s"%options.cert) cacert = os.path.join(options.confdir, "mitmproxy-ca.pem") cacert = os.path.expanduser(cacert) if not os.path.exists(cacert): utils.dummy_ca(cacert) if getattr(options, "cache", None) is not None: options.cache = os.path.expanduser(options.cache) body_size_limit = utils.parse_size(options.body_size_limit) if options.reverse_proxy: rp = utils.parse_proxy_spec(options.reverse_proxy) if not rp: parser.error("Invalid reverse proxy specification: %s"%options.reverse_proxy) else: rp = None return ProxyConfig( certfile = options.cert, cacert = cacert, ciphers = options.ciphers, cert_wait_time = options.cert_wait_time, body_size_limit = body_size_limit, upstream_cert = options.upstream_cert, reverse_proxy = rp )