From 78750a8b4da217a2b3f3eac23bea92b6c428fc35 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 10 Mar 2014 02:32:27 +0100 Subject: [PATCH 1/3] lay the foundations for --(in|out)(abs|rel) command line switches, as proposed in https://groups.google.com/forum/#!topic/mitmproxy/nApno2TXS0c --- libmproxy/protocol/http.py | 74 ++++++++++++++++++++--------------- libmproxy/proxy/config.py | 14 ++++--- libmproxy/proxy/primitives.py | 4 +- libmproxy/proxy/server.py | 18 ++------- test/test_protocol_http.py | 4 +- test/tservers.py | 13 ++++-- 6 files changed, 67 insertions(+), 60 deletions(-) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 77a09e614..c8147a3c9 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -843,6 +843,12 @@ class HttpAuthenticationError(Exception): class HTTPHandler(ProtocolHandler, TemporaryServerChangeMixin): + def __init__(self, c): + super(HTTPHandler, self).__init__(c) + self.expected_form_in = c.config.http_form_in + self.expected_form_out = c.config.http_form_out + self.skip_authentication = False + def handle_messages(self): while self.handle_flow(): pass @@ -877,13 +883,15 @@ class HTTPHandler(ProtocolHandler, TemporaryServerChangeMixin): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.change_server) try: req = HTTPRequest.from_stream(self.c.client_conn.rfile, - body_size_limit=self.c.config.body_size_limit) + body_size_limit=self.c.config.body_size_limit) self.c.log("request", [req._assemble_first_line(req.form_in)]) - self.process_request(flow, req) + send_upstream = self.process_request(flow, req) + if not send_upstream: + return True # Be careful NOT to assign the request to the flow before # process_request completes. This is because the call can raise an - # exception. If the requets object is already attached, this results + # exception. If the request object is already attached, this results # in an Error object that has an attached request that has not been # sent through to the Master. flow.request = req @@ -1004,44 +1012,48 @@ class HTTPHandler(ProtocolHandler, TemporaryServerChangeMixin): Upgrade the connection to SSL after an authority (CONNECT) request has been made. """ self.c.log("Received CONNECT request. Upgrading to SSL...") - self.c.mode = "transparent" - self.c.determine_conntype() + self.expected_form_in = "relative" + self.expected_form_out = "relative" self.c.establish_ssl(server=True, client=True) self.c.log("Upgrade to SSL completed.") - raise ConnectionTypeChange def process_request(self, flow, request): - if self.c.mode == "regular": + + if not self.skip_authentication: self.authenticate(request) - if request.form_in == "authority" and self.c.client_conn.ssl_established: - raise http.HttpError(502, "Must not CONNECT on already encrypted connection") - # If we have a CONNECT request, we might need to intercept if request.form_in == "authority": - directly_addressed_at_mitmproxy = (self.c.mode == "regular" and not self.c.config.forward_proxy) - if directly_addressed_at_mitmproxy: - self.c.set_server_address((request.host, request.port), AddressPriority.FROM_PROTOCOL) - flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow - self.c.client_conn.wfile.write( - 'HTTP/1.1 200 Connection established\r\n' + - ('Proxy-agent: %s\r\n' % self.c.server_version) + - '\r\n' - ) - self.c.client_conn.wfile.flush() - self.ssl_upgrade() # raises ConnectionTypeChange exception + if self.c.client_conn.ssl_established: + raise http.HttpError(400, "Must not CONNECT on already encrypted connection") - if self.c.mode == "regular": - if request.form_in == "authority": # forward mode - self.hook_reconnect(request) - elif request.form_in == "absolute": - if request.scheme != "http": - raise http.HttpError(400, "Invalid Request") - if not self.c.config.forward_proxy: - request.form_out = "relative" + if self.expected_form_in == "absolute": + if not self.c.config.upstream_server: self.c.set_server_address((request.host, request.port), AddressPriority.FROM_PROTOCOL) flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow - else: - raise http.HttpError(400, "Invalid request form (absolute-form or authority-form required)") + self.c.client_conn.send( + 'HTTP/1.1 200 Connection established\r\n' + + ('Proxy-agent: %s\r\n' % self.c.server_version) + + '\r\n' + ) + self.ssl_upgrade() + self.skip_authentication = True + return False + else: + self.hook_reconnect(request) + return True + elif request.form_in == self.expected_form_in: + if request.form_in == "absolute": + if request.scheme != "http": + raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme) + + self.c.set_server_address((request.host, request.port), AddressPriority.FROM_PROTOCOL) + flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow + + request.form_out = self.expected_form_out + return True + + raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" % (self.expected_form_in, + request.form_in)) def authenticate(self, request): if self.c.config.authenticator: diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 38c6ce890..248808e96 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -10,16 +10,17 @@ CONF_DIR = "~/.mitmproxy" class ProxyConfig: def __init__(self, confdir=CONF_DIR, clientcerts=None, - no_upstream_cert=False, body_size_limit=None, reverse_proxy=None, - forward_proxy=None, transparent_proxy=None, authenticator=None, + no_upstream_cert=False, body_size_limit=None, upstream_server=None, + http_form_in="absolute", http_form_out="relative", transparent_proxy=None, authenticator=None, ciphers=None, certs=None ): self.ciphers = ciphers self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit - self.reverse_proxy = reverse_proxy - self.forward_proxy = forward_proxy + self.upstream_server = upstream_server + self.http_form_in = http_form_in + self.http_form_out = http_form_out self.transparent_proxy = transparent_proxy self.authenticator = authenticator self.confdir = os.path.expanduser(confdir) @@ -93,8 +94,9 @@ def process_proxy_options(parser, options): clientcerts=options.clientcerts, body_size_limit=body_size_limit, no_upstream_cert=options.no_upstream_cert, - reverse_proxy=rp, - forward_proxy=fp, + upstream_server=(rp or fp), + http_form_in=("relative" if (rp or trans) else "absolute"), + http_form_out=("absolute" if fp else "relative"), transparent_proxy=trans, authenticator=authenticator, ciphers=options.ciphers, diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 8dd0e16a0..75ad54827 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -23,12 +23,10 @@ class AddressPriority(object): Enum that signifies the priority of the given address when choosing the destination host. Higher is better (None < i) """ - FORCE = 5 - """forward mode""" MANUALLY_CHANGED = 4 """user changed the target address in the ui""" FROM_SETTINGS = 3 - """reverse proxy mode""" + """upstream proxy from arguments (reverse proxy or forward proxy)""" FROM_CONNECTION = 2 """derived from transparent resolver""" FROM_PROTOCOL = 1 diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 37ec7758b..5aaabf878 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -59,12 +59,6 @@ class ConnectionHandler: self.conntype = None self.sni = None - self.mode = "regular" - if self.config.reverse_proxy: - self.mode = "reverse" - if self.config.transparent_proxy: - self.mode = "transparent" - def handle(self): self.log("clientconnect") self.channel.ask("clientconnect", self) @@ -76,11 +70,8 @@ class ConnectionHandler: # Can we already identify the target server and connect to it? server_address = None address_priority = None - if self.config.forward_proxy: - server_address = self.config.forward_proxy[1:] - address_priority = AddressPriority.FORCE - elif self.config.reverse_proxy: - server_address = self.config.reverse_proxy[1:] + if self.config.upstream_server: + server_address = self.config.upstream_server[1:] address_priority = AddressPriority.FROM_SETTINGS elif self.config.transparent_proxy: server_address = self.config.transparent_proxy["resolver"].original_addr( @@ -125,8 +116,8 @@ class ConnectionHandler: if self.config.transparent_proxy: client_ssl = server_ssl = (self.server_conn.address.port in self.config.transparent_proxy["sslports"]) - elif self.config.reverse_proxy: - client_ssl = server_ssl = (self.config.reverse_proxy[0] == "https") + elif self.config.upstream_server: + client_ssl = server_ssl = (self.config.upstream_server[0] == "https") # TODO: Make protocol generic (as with transparent proxies) # TODO: Add SSL-terminating capatbility (SSL -> mitmproxy -> plain and vice versa) if client_ssl or server_ssl: @@ -152,7 +143,6 @@ class ConnectionHandler: """ Sets a new server address with the given priority. Does not re-establish either connection or SSL handshake. - @type priority: libmproxy.proxy.primitives.AddressPriority """ address = tcp.Address.wrap(address) diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index 6ff0cb65a..290d82a10 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -90,7 +90,7 @@ class TestInvalidRequests(tservers.HTTPProxTest): def test_double_connect(self): p = self.pathoc() r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port)) - assert r.status_code == 502 + assert r.status_code == 400 assert "Must not CONNECT on already encrypted connection" in r.content def test_relative_request(self): @@ -98,7 +98,7 @@ class TestInvalidRequests(tservers.HTTPProxTest): p.connect() r = p.request("get:/p/200") assert r.status_code == 400 - assert "Invalid request form" in r.content + assert "Invalid HTTP request form" in r.content class TestProxyChaining(tservers.HTTPChainProxyTest): diff --git a/test/tservers.py b/test/tservers.py index bfafc8cd6..addc70110 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -197,6 +197,8 @@ class TransparentProxTest(ProxTestBase): resolver = cls.resolver(cls.server.port), sslports = ports ) + d["http_form_in"] = "relative" + d["http_form_out"] = "relative" return d def pathod(self, spec, sni=None): @@ -225,11 +227,13 @@ class ReverseProxTest(ProxTestBase): @classmethod def get_proxy_config(cls): d = ProxTestBase.get_proxy_config() - d["reverse_proxy"] = ( + d["upstream_server"] = ( "https" if cls.ssl else "http", "127.0.0.1", cls.server.port ) + d["http_form_in"] = "relative" + d["http_form_out"] = "relative" return d def pathoc(self, sni=None): @@ -258,18 +262,19 @@ class ChainProxTest(ProxTestBase): Chain n instances of mitmproxy in a row - because we can. """ n = 2 - chain_config = [lambda: ProxyConfig( - )] * n + chain_config = [lambda: ProxyConfig()] * n @classmethod def setupAll(cls): super(ChainProxTest, cls).setupAll() cls.chain = [] for i in range(cls.n): config = cls.chain_config[i]() - config.forward_proxy = ("http", "127.0.0.1", + config.upstream_server = ("http", "127.0.0.1", cls.proxy.port if i == 0 else cls.chain[-1].port ) + config.http_form_in = "absolute" + config.http_form_out = "absolute" tmaster = cls.masterclass(config) tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp) cls.chain.append(ProxyThread(tmaster)) From fe58c1c6eb16fdc14bd24843cb896b3d8a4eefc8 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 10 Mar 2014 05:11:51 +0100 Subject: [PATCH 2/3] add advanced proxying options, add SSL-terminating capability to mitmproxy --- doc-src/features/reverseproxy.html | 2 +- libmproxy/cmdline.py | 79 ++++++++++++++++++++++++------ libmproxy/console/__init__.py | 25 +--------- libmproxy/console/help.py | 1 - libmproxy/protocol/http.py | 2 +- libmproxy/proxy/config.py | 58 +++++++++++----------- libmproxy/proxy/primitives.py | 41 ++++++++++++++-- libmproxy/proxy/server.py | 42 +++------------- libmproxy/utils.py | 9 ---- test/test_cmdline.py | 11 +++++ test/test_proxy.py | 26 +++------- test/test_utils.py | 8 --- test/tservers.py | 31 ++++++------ test/tutils.py | 11 ++++- 14 files changed, 182 insertions(+), 164 deletions(-) diff --git a/doc-src/features/reverseproxy.html b/doc-src/features/reverseproxy.html index d399cdc0c..7be02b876 100644 --- a/doc-src/features/reverseproxy.html +++ b/doc-src/features/reverseproxy.html @@ -10,7 +10,7 @@ __Host__ header field from the request, not the reverse proxy server. - + diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 72c137698..6c0cae9f3 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,12 +1,16 @@ from . import proxy import re, filt import argparse +from argparse import ArgumentTypeError +from netlib import http APP_HOST = "mitm.it" APP_PORT = 80 -class ParseException(Exception): pass -class OptionException(Exception): pass + +class ParseException(Exception): + pass + def _parse_hook(s): sep, rem = s[0], s[1:] @@ -91,6 +95,26 @@ def parse_setheader(s): return _parse_hook(s) +def parse_server_spec(url): + + normalized_url = re.sub("^https?2", "", url) + + p = http.parse_url(normalized_url) + if not p or not p[1]: + raise ArgumentTypeError("Invalid server specification: %s" % url) + + if url.lower().startswith("https2http"): + ssl = [True, False] + elif url.lower().startswith("http2https"): + ssl = [False, True] + elif url.lower().startswith("https"): + ssl = [True, True] + else: + ssl = [False, False] + + return ssl + list(p[1:3]) + + def get_common_options(options): stickycookie, stickyauth = None, None if options.stickycookie_filt: @@ -104,17 +128,17 @@ def get_common_options(options): try: p = parse_replace_hook(i) except ParseException, e: - raise OptionException(e.message) + raise ArgumentTypeError(e.message) reps.append(p) for i in options.replace_file: try: patt, rex, path = parse_replace_hook(i) except ParseException, e: - raise OptionException(e.message) + raise ArgumentTypeError(e.message) try: v = open(path, "rb").read() except IOError, e: - raise OptionException("Could not read replace file: %s"%path) + raise ArgumentTypeError("Could not read replace file: %s"%path) reps.append((patt, rex, v)) @@ -123,7 +147,7 @@ def get_common_options(options): try: p = parse_setheader(i) except ParseException, e: - raise OptionException(e.message) + raise ArgumentTypeError(e.message) setheaders.append(p) return dict( @@ -185,16 +209,48 @@ def common_options(parser): action="store", type = int, dest="port", default=8080, help = "Proxy service port." ) + # We could make a mutually exclusive group out of -R, -F, -T, but we don't do because + # - --upstream-server should be in that group as well, but it's already in a different group. + # - our own error messages are more helpful parser.add_argument( - "-P", - action="store", dest="reverse_proxy", default=None, + "-R", + action="store", type=parse_server_spec, dest="reverse_proxy", default=None, help="Reverse proxy to upstream server: http[s]://host[:port]" ) parser.add_argument( "-F", - action="store", dest="forward_proxy", default=None, + action="store", type=parse_server_spec, dest="forward_proxy", default=None, help="Proxy to unconditionally forward to: http[s]://host[:port]" ) + parser.add_argument( + "-T", + action="store_true", dest="transparent_proxy", default=False, + help="Set transparent proxy mode." + ) + + group = parser.add_argument_group( + "Advanced Proxy Options", + """ + The following options allow a custom adjustment of the proxy behavior. + Normally, you don't want to use these options directly and use the provided wrappers instead (-R, -F, -T). + """.strip() + ) + group.add_argument( + "--http-form-in", dest="http_form_in", default=None, + action="store", choices=("relative", "absolute"), + help="Override the HTTP request form accepted by the proxy" + ) + group.add_argument( + "--http-form-out", dest="http_form_out", default=None, + action="store", choices=("relative", "absolute"), + help="Override the HTTP request form sent upstream by the proxy" + ) + group.add_argument( + "--upstream-server", dest="manual_upstream_server", default=None, + action="store", type=parse_server_spec, + help="Override the destination server all requests are sent to." + ) + parser.add_argument( "-q", action="store_true", dest="quiet", @@ -216,11 +272,6 @@ def common_options(parser): action="store", dest="stickycookie_filt", default=None, metavar="FILTER", help="Set sticky cookie filter. Matched against requests." ) - parser.add_argument( - "-T", - action="store_true", dest="transparent_proxy", default=False, - help="Set transparent proxy mode." - ) parser.add_argument( "-u", action="store", dest="stickyauth_filt", default=None, metavar="FILTER", diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 4a58e7711..846abba76 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -144,10 +144,6 @@ class StatusBar(common.WWrap): r.append("[") r.append(("heading_key", "u")) r.append(":%s]"%self.master.stickyauth_txt) - if self.master.server.config.reverse_proxy: - r.append("[") - r.append(("heading_key", "P")) - r.append(":%s]"%utils.unparse_url(*self.master.server.config.reverse_proxy)) if self.master.state.default_body_view.name != "Auto": r.append("[") r.append(("heading_key", "M")) @@ -172,6 +168,8 @@ class StatusBar(common.WWrap): if opts: r.append("[%s]"%(":".join(opts))) + if self.master.server.config.upstream_server: + r.append("[dest:%s]"%utils.unparse_url(*self.master.server.config.upstream_server)) if self.master.scripts: r.append("[scripts:%s]"%len(self.master.scripts)) if self.master.debug: @@ -763,15 +761,6 @@ class ConsoleMaster(flow.FlowMaster): self.state.default_body_view = v self.refresh_focus() - def set_reverse_proxy(self, txt): - if not txt: - self.server.config.reverse_proxy = None - else: - s = utils.parse_proxy_spec(txt) - if not s: - return "Invalid reverse proxy specification" - self.server.config.reverse_proxy = s - def drawscreen(self): size = self.ui.get_cols_rows() canvas = self.view.render(size, focus=1) @@ -866,16 +855,6 @@ class ConsoleMaster(flow.FlowMaster): contentview.view_prompts, self.change_default_display_mode ) - elif k == "P": - if self.server.config.reverse_proxy: - p = utils.unparse_url(*self.server.config.reverse_proxy) - else: - p = "" - self.prompt( - "Reverse proxy: ", - p, - self.set_reverse_proxy - ) elif k == "R": self.view_grideditor( grideditor.ReplaceEditor( diff --git a/libmproxy/console/help.py b/libmproxy/console/help.py index 0d01ac6f7..b0b7c48cb 100644 --- a/libmproxy/console/help.py +++ b/libmproxy/console/help.py @@ -109,7 +109,6 @@ class HelpView(urwid.ListBox): ("q", "quit / return to flow list"), ("Q", "quit without confirm prompt"), - ("P", "set reverse proxy mode"), ("R", "edit replacement patterns"), ("s", "set/unset script"), ("S", "server replay"), diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index c8147a3c9..aff4a8179 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1027,7 +1027,7 @@ class HTTPHandler(ProtocolHandler, TemporaryServerChangeMixin): raise http.HttpError(400, "Must not CONNECT on already encrypted connection") if self.expected_form_in == "absolute": - if not self.c.config.upstream_server: + if not self.c.config.get_upstream_server: self.c.set_server_address((request.host, request.port), AddressPriority.FROM_PROTOCOL) flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow self.c.client_conn.send( diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 248808e96..ae24d4c9a 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,7 +1,7 @@ import os from .. import utils, platform from netlib import http_auth, certutils - +from .primitives import ConstUpstreamServerResolver, TransparentUpstreamServerResolver TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" @@ -10,18 +10,17 @@ CONF_DIR = "~/.mitmproxy" class ProxyConfig: def __init__(self, confdir=CONF_DIR, clientcerts=None, - no_upstream_cert=False, body_size_limit=None, upstream_server=None, - http_form_in="absolute", http_form_out="relative", transparent_proxy=None, authenticator=None, + no_upstream_cert=False, body_size_limit=None, get_upstream_server=None, + http_form_in="absolute", http_form_out="relative", authenticator=None, ciphers=None, certs=None ): self.ciphers = ciphers self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit - self.upstream_server = upstream_server + self.get_upstream_server = get_upstream_server self.http_form_in = http_form_in self.http_form_out = http_form_out - self.transparent_proxy = transparent_proxy self.authenticator = authenticator self.confdir = os.path.expanduser(confdir) self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) @@ -29,32 +28,34 @@ class ProxyConfig: def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) - if options.reverse_proxy and options.transparent_proxy: - return parser.error("Can't set both reverse proxy and transparent proxy.") + c = 0 + http_form_in, http_form_out = "absolute", "relative" + get_upstream_server = None if options.transparent_proxy: + c += 1 if not platform.resolver: return parser.error("Transparent mode not supported on this platform.") - trans = dict( - resolver=platform.resolver(), - sslports=TRANSPARENT_SSL_PORTS - ) - else: - trans = None - + get_upstream_server = TransparentUpstreamServerResolver(platform.resolver(), TRANSPARENT_SSL_PORTS) + http_form_in, http_form_out = "relative", "relative" if options.reverse_proxy: - rp = utils.parse_proxy_spec(options.reverse_proxy) - if not rp: - return parser.error("Invalid reverse proxy specification: %s" % options.reverse_proxy) - else: - rp = None - + c += 1 + get_upstream_server = ConstUpstreamServerResolver(options.reverse_proxy) + http_form_in, http_form_out = "relative", "relative" if options.forward_proxy: - fp = utils.parse_proxy_spec(options.forward_proxy) - if not fp: - return parser.error("Invalid forward proxy specification: %s" % options.forward_proxy) - else: - fp = None + c += 1 + get_upstream_server = ConstUpstreamServerResolver(options.forward_proxy) + http_form_in, http_form_out = "absolute", "absolute" + if options.manual_upstream_server: + c += 1 + get_upstream_server = ConstUpstreamServerResolver(options.manual_upstream_server) + if c > 1: + return parser.error("Transparent mode, reverse mode, forward mode and " + "specification of an upstream server are mutually exclusive.") + if options.http_form_in: + http_form_in = options.http_form_in + if options.http_form_out: + http_form_out = options.http_form_out if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) @@ -94,10 +95,9 @@ def process_proxy_options(parser, options): clientcerts=options.clientcerts, body_size_limit=body_size_limit, no_upstream_cert=options.no_upstream_cert, - upstream_server=(rp or fp), - http_form_in=("relative" if (rp or trans) else "absolute"), - http_form_out=("absolute" if fp else "relative"), - transparent_proxy=trans, + get_upstream_server=get_upstream_server, + http_form_in=http_form_in, + http_form_out=http_form_out, authenticator=authenticator, ciphers=options.ciphers, certs = certs, diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 75ad54827..e49d9bb4a 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -18,17 +18,48 @@ class ProxyServerError(Exception): pass +class UpstreamServerResolver(object): + def __call__(self, conn): + """ + Returns the address of the server to connect to. + """ + raise NotImplementedError + + +class ConstUpstreamServerResolver(UpstreamServerResolver): + def __init__(self, dst): + self.dst = dst + + def __call__(self, conn): + return self.dst + + +class TransparentUpstreamServerResolver(UpstreamServerResolver): + def __init__(self, resolver, sslports): + self.resolver = resolver + self.sslports = sslports + + def __call__(self, conn): + dst = self.resolver.original_addr(conn) + if not dst: + raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") + + if dst[1] in self.sslports: + ssl = True + else: + ssl = False + return [ssl, ssl] + list(dst) + + class AddressPriority(object): """ Enum that signifies the priority of the given address when choosing the destination host. Higher is better (None < i) """ - MANUALLY_CHANGED = 4 + MANUALLY_CHANGED = 3 """user changed the target address in the ui""" - FROM_SETTINGS = 3 - """upstream proxy from arguments (reverse proxy or forward proxy)""" - FROM_CONNECTION = 2 - """derived from transparent resolver""" + FROM_SETTINGS = 2 + """upstream server from arguments (reverse proxy, forward proxy or from transparent resolver)""" FROM_PROTOCOL = 1 """derived from protocol (e.g. absolute-form http requests)""" diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 5aaabf878..554ee551b 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -68,22 +68,13 @@ class ConnectionHandler: try: try: # Can we already identify the target server and connect to it? - server_address = None - address_priority = None - if self.config.upstream_server: - server_address = self.config.upstream_server[1:] - address_priority = AddressPriority.FROM_SETTINGS - elif self.config.transparent_proxy: - server_address = self.config.transparent_proxy["resolver"].original_addr( - self.client_conn.connection) - if not server_address: - raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") - address_priority = AddressPriority.FROM_CONNECTION - self.log("transparent to %s:%s" % server_address) - - if server_address: - self.set_server_address(server_address, address_priority) - self._handle_ssl() + if self.config.get_upstream_server: + upstream_info = self.config.get_upstream_server(self.client_conn.connection) + self.set_server_address(upstream_info[2:], AddressPriority.FROM_SETTINGS) + client_ssl, server_ssl = upstream_info[:2] + if client_ssl or server_ssl: + self.establish_server_connection() + self.establish_ssl(client=client_ssl, server=server_ssl) while not self.close: try: @@ -105,25 +96,6 @@ class ConnectionHandler: self.log("clientdisconnect") self.channel.tell("clientdisconnect", self) - def _handle_ssl(self): - """ - Helper function of .handle() - Check if we can already identify SSL connections. - If so, connect to the server and establish an SSL connection - """ - client_ssl = False - server_ssl = False - - if self.config.transparent_proxy: - client_ssl = server_ssl = (self.server_conn.address.port in self.config.transparent_proxy["sslports"]) - elif self.config.upstream_server: - client_ssl = server_ssl = (self.config.upstream_server[0] == "https") - # TODO: Make protocol generic (as with transparent proxies) - # TODO: Add SSL-terminating capatbility (SSL -> mitmproxy -> plain and vice versa) - if client_ssl or server_ssl: - self.establish_server_connection() - self.establish_ssl(client=client_ssl, server=server_ssl) - def del_server_connection(self): """ Deletes an existing server connection. diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 1ec4685de..17ad6f091 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -1,7 +1,6 @@ import os, datetime, urllib, re import time, functools, cgi import json -from netlib import http def timestamp(): """ @@ -143,14 +142,6 @@ class LRUCache: return ret return wrap - -def parse_proxy_spec(url): - p = http.parse_url(url) - if not p or not p[1]: - return None - return p[:3] - - def parse_content_type(c): """ A simple parser for content-type values. Returns a (type, subtype, diff --git a/test/test_cmdline.py b/test/test_cmdline.py index dbc61bfc2..12e8aa897 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -36,6 +36,17 @@ def test_parse_replace_hook(): ) +def test_parse_server_spec(): + tutils.raises("Invalid server specification", cmdline.parse_server_spec, "") + assert cmdline.parse_server_spec("http://foo.com:88") == [False, False, "foo.com", 88] + assert cmdline.parse_server_spec("http://foo.com") == [False, False, "foo.com", 80] + assert cmdline.parse_server_spec("https://foo.com") == [True, True, "foo.com", 443] + assert cmdline.parse_server_spec("https2http://foo.com") == [True, False, "foo.com", 80] + assert cmdline.parse_server_spec("http2https://foo.com") == [False, True, "foo.com", 443] + tutils.raises("Invalid server specification", cmdline.parse_server_spec, "foo.com") + tutils.raises("Invalid server specification", cmdline.parse_server_spec, "http://") + + def test_parse_setheaders(): x = cmdline.parse_setheader("/foo/bar/voing") assert x == ("foo", "bar", "voing") diff --git a/test/test_proxy.py b/test/test_proxy.py index f53aa7621..0f4384824 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -43,28 +43,15 @@ class TestServerConnection: sc.finish() -class MockParser: - def __init__(self): - self.err = None - - def error(self, e): - self.err = e - - def __repr__(self): - return "ParseError(%s)"%self.err - - class TestProcessProxyOptions: def p(self, *args): - parser = argparse.ArgumentParser() + parser = tutils.MockParser() cmdline.common_options(parser) opts = parser.parse_args(args=args) - m = MockParser() - return m, process_proxy_options(m, opts) + return parser, process_proxy_options(parser, opts) def assert_err(self, err, *args): - m, p = self.p(*args) - assert err.lower() in m.err.lower() + tutils.raises(err, self.p, *args) def assert_noerr(self, *args): m, p = self.p(*args) @@ -84,11 +71,10 @@ class TestProcessProxyOptions: @mock.patch("libmproxy.platform.resolver") def test_transparent_reverse(self, o): - self.assert_err("can't set both", "-P", "reverse", "-T") + self.assert_err("mutually exclusive", "-R", "http://localhost", "-T") self.assert_noerr("-T") - assert o.call_count == 1 - self.assert_err("invalid reverse proxy", "-P", "reverse") - self.assert_noerr("-P", "http://localhost") + self.assert_err("Invalid server specification", "-R", "reverse") + self.assert_noerr("-R", "http://localhost") def test_client_certs(self): with tutils.tmpdir() as confdir: diff --git a/test/test_utils.py b/test/test_utils.py index f40e0a31c..11c26dba8 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -87,14 +87,6 @@ def test_LRUCache(): assert len(f._cachelist_one) == 2 -def test_parse_proxy_spec(): - assert not utils.parse_proxy_spec("") - assert utils.parse_proxy_spec("http://foo.com:88") == ("http", "foo.com", 88) - assert utils.parse_proxy_spec("http://foo.com") == ("http", "foo.com", 80) - assert not utils.parse_proxy_spec("foo.com") - assert not utils.parse_proxy_spec("http://") - - def test_unparse_url(): assert utils.unparse_url("http", "foo.com", 99, "") == "http://foo.com:99" assert utils.unparse_url("http", "foo.com", 80, "") == "http://foo.com" diff --git a/test/tservers.py b/test/tservers.py index addc70110..9ad3b735d 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -4,6 +4,7 @@ import shutil, tempfile import flask from libmproxy.proxy.config import ProxyConfig from libmproxy.proxy.server import ProxyServer +from libmproxy.proxy.primitives import TransparentUpstreamServerResolver import libpathod.test, libpathod.pathoc from libmproxy import flow, controller from libmproxy.cmdline import APP_HOST, APP_PORT @@ -193,10 +194,7 @@ class TransparentProxTest(ProxTestBase): ports = [cls.server.port, cls.server2.port] else: ports = [] - d["transparent_proxy"] = dict( - resolver = cls.resolver(cls.server.port), - sslports = ports - ) + d["get_upstream_server"] = TransparentUpstreamServerResolver(cls.resolver(cls.server.port), ports) d["http_form_in"] = "relative" d["http_form_out"] = "relative" return d @@ -227,11 +225,12 @@ class ReverseProxTest(ProxTestBase): @classmethod def get_proxy_config(cls): d = ProxTestBase.get_proxy_config() - d["upstream_server"] = ( - "https" if cls.ssl else "http", - "127.0.0.1", - cls.server.port - ) + d["get_upstream_server"] = lambda c: ( + True if cls.ssl else False, + True if cls.ssl else False, + "127.0.0.1", + cls.server.port + ) d["http_form_in"] = "relative" d["http_form_out"] = "relative" return d @@ -262,19 +261,17 @@ class ChainProxTest(ProxTestBase): Chain n instances of mitmproxy in a row - because we can. """ n = 2 - chain_config = [lambda: ProxyConfig()] * n + chain_config = [lambda port: ProxyConfig( + get_upstream_server = lambda c: (False, False, "127.0.0.1", port), + http_form_in = "absolute", + http_form_out = "absolute" + )] * n @classmethod def setupAll(cls): super(ChainProxTest, cls).setupAll() cls.chain = [] for i in range(cls.n): - config = cls.chain_config[i]() - config.upstream_server = ("http", "127.0.0.1", - cls.proxy.port if i == 0 else - cls.chain[-1].port - ) - config.http_form_in = "absolute" - config.http_form_out = "absolute" + config = cls.chain_config[i](cls.proxy.port if i == 0 else cls.chain[-1].port) tmaster = cls.masterclass(config) tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp) cls.chain.append(ProxyThread(tmaster)) diff --git a/test/tutils.py b/test/tutils.py index 3f6592b0e..c527a64af 100644 --- a/test/tutils.py +++ b/test/tutils.py @@ -1,4 +1,4 @@ -import os, shutil, tempfile +import os, shutil, tempfile, argparse from contextlib import contextmanager from libmproxy import flow, utils, controller from libmproxy.protocol import http @@ -136,6 +136,15 @@ def tmpdir(*args, **kwargs): shutil.rmtree(temp_workdir) +class MockParser(argparse.ArgumentParser): + """ + argparse.ArgumentParser sys.exits() by default. + Make it more testable by throwing an exception instead. + """ + def error(self, message): + raise Exception(message) + + def raises(exc, obj, *args, **kwargs): """ Assert that a callable raises a specified exception. From 9cc10630c8e0b0f249ed852299d3316ba42f128d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 10 Mar 2014 17:01:30 +0100 Subject: [PATCH 3/3] update docs, fix #215 --- doc-src/scripting/inlinescripts.html | 28 ++++++++++++++-------------- libmproxy/cmdline.py | 2 +- libmproxy/console/flowview.py | 4 ++-- libmproxy/protocol/__init__.py | 2 +- libmproxy/proxy/server.py | 20 +++++++++++--------- 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/doc-src/scripting/inlinescripts.html b/doc-src/scripting/inlinescripts.html index 7ab1c1013..32a98e99b 100644 --- a/doc-src/scripting/inlinescripts.html +++ b/doc-src/scripting/inlinescripts.html @@ -76,25 +76,25 @@ The main classes you will deal with in writing mitmproxy scripts are:
command-line -P http[s]://hostname[:port]command-line -R http[s]://hostname[:port]
mitmproxy shortcut P
- + + + + + - - - + + + - + - + - - - - @@ -103,15 +103,15 @@ The main classes you will deal with in writing mitmproxy scripts are: calls (used mainly for headers). - + - + - + @@ -124,7 +124,7 @@ The canonical API documentation is the code. You can view the API documentation using pydoc (which is installed with Python by default), like this: 
-> pydoc libmproxy.flow.Request
+> pydoc libmproxy.protocol.http.HTTPRequest
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 6c0cae9f3..bee4aa608 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -209,7 +209,7 @@ def common_options(parser): action="store", type = int, dest="port", default=8080, help = "Proxy service port." ) - # We could make a mutually exclusive group out of -R, -F, -T, but we don't do because + # We could make a mutually exclusive group out of -R, -F, -T, but we don't do that because # - --upstream-server should be in that group as well, but it's already in a different group. # - our own error messages are more helpful parser.add_argument( diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index f5b5f83f4..9b6368405 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -2,7 +2,7 @@ import os, sys, copy import urwid import common, grideditor, contentview from .. import utils, flow, controller -from ..protocol.http import CONTENT_MISSING +from ..protocol.http import HTTPResponse, CONTENT_MISSING class SearchError(Exception): pass @@ -571,7 +571,7 @@ class FlowView(common.WWrap): conn = self.flow.request else: if not self.flow.response: - self.flow.response = flow.Response( + self.flow.response = HTTPResponse( self.flow.request, self.flow.request.httpversion, 200, "OK", flow.ODictCaseless(), "", None diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index 6200757f8..b253fbd5c 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1,4 +1,4 @@ -from libmproxy.proxy.primitives import AddressPriority +from ..proxy.primitives import AddressPriority KILL = 0 # const for killed requests diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 554ee551b..c77ab2a81 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -1,10 +1,9 @@ import socket -from .. import version, protocol -from libmproxy.proxy.primitives import Log -from .primitives import ProxyServerError -from .connection import ClientConnection, ServerConnection -from .primitives import ProxyError, ConnectionTypeChange, AddressPriority +from OpenSSL import SSL from netlib import tcp +from .primitives import ProxyServerError, Log, ProxyError, ConnectionTypeChange, AddressPriority +from .connection import ClientConnection, ServerConnection +from .. import version, protocol class DummyServer: @@ -23,6 +22,7 @@ class DummyServer: class ProxyServer(tcp.TCPServer): allow_reuse_address = True bound = True + def __init__(self, config, port, host='', server_version=version.NAMEVERSION): """ Raises ProxyServerError if there's a startup problem. @@ -51,8 +51,11 @@ class ProxyServer(tcp.TCPServer): class ConnectionHandler: def __init__(self, config, client_connection, client_address, server, channel, server_version): self.config = config + """@type: libmproxy.proxy.config.ProxyConfig""" self.client_conn = ClientConnection(client_connection, client_address, server) + """@type: libmproxy.proxy.connection.ClientConnection""" self.server_conn = None + """@type: libmproxy.proxy.connection.ServerConnection""" self.channel, self.server_version = channel, server_version self.close = False @@ -98,7 +101,7 @@ class ConnectionHandler: def del_server_connection(self): """ - Deletes an existing server connection. + Deletes (and closes) an existing server connection. """ if self.server_conn and self.server_conn.connection: self.server_conn.finish() @@ -150,8 +153,7 @@ class ConnectionHandler: """ Establishes SSL on the existing connection(s) to the server or the client, as specified by the parameters. If the target server is on the pass-through list, - the conntype attribute will be changed and the SSL connection won't be wrapped. - A protocol handler must raise a ConnTypeChanged exception if it detects that this is happening + the conntype attribute will be changed and a ConnTypeChanged exception will be raised. """ # TODO: Implement SSL pass-through handling and change conntype passthrough = [ @@ -160,7 +162,7 @@ class ConnectionHandler: ] if self.server_conn.address.host in passthrough or self.sni in passthrough: self.conntype = "tcp" - return + raise ConnectionTypeChange # Logging if client or server:
libmproxy.flow.ClientConnectionlibmproxy.proxy.server.ConnectionHandlerDescribes a proxy client connection session. Always has a client_conn attribute, might have a server_conn attribute.
libmproxy.proxy.connection.ClientConnection Describes a client connection.
libmproxy.flow.ClientDisconnectionDescribes a client disconnection.
libmproxy.proxy.connection.ServerConnectionDescribes a server connection.
libmproxy.flow.Errorlibmproxy.protocol.primitives.Error A communications error.
libmproxy.flow.Flowlibmproxy.protocol.http.HTTPFlow A collection of objects representing a single HTTP transaction.
libmproxy.flow.HeadersHTTP headers for a request or response.
libmproxy.flow.ODict
libmproxy.flow.Responselibmproxy.protocol.http.HTTPResponse An HTTP response.
libmproxy.flow.Requestlibmproxy.protocol.http.HTTPRequest An HTTP request.
libmproxy.flow.ScriptContextlibmproxy.script.ScriptContext A handle for interacting with mitmproxy's from within scripts.