From 1cc48345e13917aadc1e0fd93d6011139e78e3d9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 28 Aug 2015 01:51:13 +0200 Subject: [PATCH] clean up config/cmdline, fix bugs, remove cruft --- libmproxy/cmdline.py | 253 ++++++++++++++++----------- libmproxy/flow.py | 6 +- libmproxy/protocol/http.py | 2 +- libmproxy/protocol2/__init__.py | 7 +- libmproxy/protocol2/reverse_proxy.py | 5 +- libmproxy/protocol2/root_context.py | 10 +- libmproxy/protocol2/socks_proxy.py | 2 +- libmproxy/protocol2/tls.py | 38 +++- libmproxy/proxy/config.py | 192 ++++++-------------- libmproxy/proxy/server.py | 38 +++- test/test_cmdline.py | 14 +- test/test_flow.py | 8 +- test/test_proxy.py | 8 +- test/test_server.py | 53 +----- test/tservers.py | 14 +- 15 files changed, 304 insertions(+), 346 deletions(-) diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index d033fb768..1d897717a 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -2,8 +2,8 @@ from __future__ import absolute_import import os import re import configargparse +from netlib.tcp import Address -from netlib import http import netlib.utils from . import filt, utils, version @@ -102,32 +102,22 @@ def parse_setheader(s): return _parse_hook(s) -def parse_server_spec(url): +def parse_server_spec(url, allowed_schemes=("http", "https")): p = netlib.utils.parse_url(url) - if not p or not p[1] or p[0] not in ("http", "https"): + if not p or not p[1] or p[0] not in allowed_schemes: raise configargparse.ArgumentTypeError( "Invalid server specification: %s" % url ) - - if p[0].lower() == "https": - ssl = [True, True] - else: - ssl = [False, False] - - return ssl + list(p[1:3]) + address = Address(p[1:3]) + scheme = p[0].lower() + return config.ServerSpec(scheme, address) def parse_server_spec_special(url): """ Provides additional support for http2https and https2http schemes. """ - normalized_url = re.sub("^https?2", "", url) - ret = parse_server_spec(normalized_url) - if url.lower().startswith("https2http"): - ret[0] = True - elif url.lower().startswith("http2https"): - ret[0] = False - return ret + return parse_server_spec(url, allowed_schemes=("http", "https", "http2https", "https2http")) def get_common_options(options): @@ -192,24 +182,24 @@ def get_common_options(options): outfile=options.outfile, verbosity=options.verbose, nopop=options.nopop, - replay_ignore_content = options.replay_ignore_content, - replay_ignore_params = options.replay_ignore_params, - replay_ignore_payload_params = options.replay_ignore_payload_params, - replay_ignore_host = options.replay_ignore_host + replay_ignore_content=options.replay_ignore_content, + replay_ignore_params=options.replay_ignore_params, + replay_ignore_payload_params=options.replay_ignore_payload_params, + replay_ignore_host=options.replay_ignore_host ) -def common_options(parser): +def basic_options(parser): parser.add_argument( '--version', - action= 'version', - version= "%(prog)s" + " " + version.VERSION + action='version', + version="%(prog)s" + " " + version.VERSION ) parser.add_argument( '--shortversion', - action= 'version', - help = "show program's short version number and exit", - version = version.VERSION + action='version', + help="show program's short version number and exit", + version=version.VERSION ) parser.add_argument( "--anticache", @@ -301,11 +291,42 @@ def common_options(parser): """ ) + +def proxy_modes(parser): + group = parser.add_argument_group("Proxy Modes").add_mutually_exclusive_group() + group.add_argument( + "-R", "--reverse", + action="store", + type=parse_server_spec_special, + dest="reverse_proxy", + default=None, + help=""" + Forward all requests to upstream HTTP server: + http[s][2http[s]]://host[:port] + """ + ) + group.add_argument( + "--socks", + action="store_true", dest="socks_proxy", default=False, + help="Set SOCKS5 proxy mode." + ) + group.add_argument( + "-T", "--transparent", + action="store_true", dest="transparent_proxy", default=False, + help="Set transparent proxy mode." + ) + group.add_argument( + "-U", "--upstream", + action="store", + type=parse_server_spec, + dest="upstream_proxy", + default=None, + help="Forward all requests to upstream proxy server: http://host[:port]" + ) + + +def proxy_options(parser): group = parser.add_argument_group("Proxy Options") - # We could make a mutually exclusive group out of -R, -U, -T, but we don't - # do that because - --upstream-server should be in that group as well, but - # it's already in a different group. - our own error messages are more - # helpful group.add_argument( "-b", "--bind-address", action="store", type=str, dest="addr", default='', @@ -344,70 +365,78 @@ def common_options(parser): action="store", type=int, dest="port", default=8080, help="Proxy service port." ) + + +def proxy_ssl_options(parser): + # TODO: Agree to consistently either use "upstream" or "server". + group = parser.add_argument_group("SSL") group.add_argument( - "-R", "--reverse", - action="store", - type=parse_server_spec_special, - dest="reverse_proxy", - default=None, - help=""" - Forward all requests to upstream HTTP server: - http[s][2http[s]]://host[:port] - """ + "--cert", + dest='certs', + default=[], + type=str, + metavar="SPEC", + action="append", + help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' + 'The domain may include a wildcard, and is equal to "*" if not specified. ' + 'The file at path is a certificate in PEM format. If a private key is included ' + 'in the PEM, it is used, else the default key in the conf dir is used. ' + 'The PEM file should contain the full certificate chain, with the leaf certificate ' + 'as the first entry. Can be passed multiple times.') + group.add_argument( + "--ciphers-client", action="store", + type=str, dest="ciphers_client", default=config.DEFAULT_CLIENT_CIPHERS, + help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( - "--socks", - action="store_true", dest="socks_proxy", default=False, - help="Set SOCKS5 proxy mode." + "--ciphers-server", action="store", + type=str, dest="ciphers_server", default=None, + help="Set supported ciphers for server connections. (OpenSSL Syntax)" ) group.add_argument( - "-T", "--transparent", - action="store_true", dest="transparent_proxy", default=False, - help="Set transparent proxy mode." + "--client-certs", action="store", + type=str, dest="clientcerts", default=None, + help="Client certificate directory." ) group.add_argument( - "-U", "--upstream", - action="store", - type=parse_server_spec, - dest="upstream_proxy", - default=None, - help="Forward all requests to upstream proxy server: http://host[:port]" + "--no-upstream-cert", default=False, + action="store_true", dest="no_upstream_cert", + help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--spoof", - action="store_true", dest="spoof_mode", default=False, - help="Use Host header to connect to HTTP servers." + "--verify-upstream-cert", default=False, + action="store_true", dest="ssl_verify_upstream_cert", + help="Verify upstream server SSL/TLS certificates and fail if invalid " + "or not present." ) group.add_argument( - "--ssl-spoof", - action="store_true", dest="ssl_spoof_mode", default=False, - help="Use TLS SNI to connect to HTTPS servers." + "--upstream-trusted-cadir", default=None, action="store", + dest="ssl_verify_upstream_trusted_cadir", + help="Path to a directory of trusted CA certificates for upstream " + "server verification prepared using the c_rehash tool." ) group.add_argument( - "--spoofed-port", - action="store", dest="spoofed_ssl_port", type=int, default=443, - help="Port number of upstream HTTPS servers in SSL spoof mode." + "--upstream-trusted-ca", default=None, action="store", + dest="ssl_verify_upstream_trusted_ca", + help="Path to a PEM formatted trusted CA certificate." + ) + group.add_argument( + "--ssl-version-client", dest="ssl_version_client", + default="secure", action="store", + choices=config.sslversion_choices.keys(), + help="Set supported SSL/TLS version for client connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." + ) + group.add_argument( + "--ssl-version-server", dest="ssl_version_server", + default="secure", action="store", + choices=config.sslversion_choices.keys(), + help="Set supported SSL/TLS version for server connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) - group = parser.add_argument_group( - "Advanced Proxy Options", - """ - The following options allow a custom adjustment of the proxy - behavior. Normally, you don't want to use these options directly and - use the provided wrappers instead (-R, -U, -T). - """ - ) - group.add_argument( - "--http-form-in", dest="http_form_in", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form accepted by the proxy" - ) - group.add_argument( - "--http-form-out", dest="http_form_out", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form sent upstream by the proxy" - ) +def onboarding_app(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( "--noapp", @@ -433,6 +462,8 @@ def common_options(parser): help="Port to serve the onboarding app from." ) + +def client_replay(parser): group = parser.add_argument_group("Client Replay") group.add_argument( "-c", "--client-replay", @@ -440,6 +471,8 @@ def common_options(parser): help="Replay client requests from a saved file." ) + +def server_replay(parser): group = parser.add_argument_group("Server Replay") group.add_argument( "-S", "--server-replay", @@ -504,6 +537,8 @@ def common_options(parser): default=False, help="Ignore request's destination host while searching for a saved flow to replay") + +def replacements(parser): group = parser.add_argument_group( "Replacements", """ @@ -520,14 +555,16 @@ def common_options(parser): ) group.add_argument( "--replace-from-file", - action = "append", type=str, dest="replace_file", default=[], - metavar = "PATH", - help = """ + action="append", type=str, dest="replace_file", default=[], + metavar="PATH", + help=""" Replacement pattern, where the replacement clause is a path to a file. """ ) + +def set_headers(parser): group = parser.add_argument_group( "Set Headers", """ @@ -543,21 +580,22 @@ def common_options(parser): help="Header set pattern." ) + +def proxy_authentication(parser): group = parser.add_argument_group( "Proxy Authentication", """ Specify which users are allowed to access the proxy and the method used for authenticating them. """ - ) - user_specification_group = group.add_mutually_exclusive_group() - user_specification_group.add_argument( + ).add_mutually_exclusive_group() + group.add_argument( "--nonanonymous", action="store_true", dest="auth_nonanonymous", help="Allow access to any user long as a credentials are specified." ) - user_specification_group.add_argument( + group.add_argument( "--singleuser", action="store", dest="auth_singleuser", type=str, metavar="USER", @@ -566,14 +604,25 @@ def common_options(parser): username:password. """ ) - user_specification_group.add_argument( + group.add_argument( "--htpasswd", action="store", dest="auth_htpasswd", type=str, metavar="PATH", help="Allow access to users specified in an Apache htpasswd file." ) - config.ssl_option_group(parser) + +def common_options(parser): + basic_options(parser) + proxy_modes(parser) + proxy_options(parser) + proxy_ssl_options(parser) + onboarding_app(parser) + client_replay(parser) + server_replay(parser) + replacements(parser) + set_headers(parser) + proxy_authentication(parser) def mitmproxy(): @@ -583,13 +632,13 @@ def mitmproxy(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmproxy.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( @@ -628,20 +677,20 @@ def mitmproxy(): def mitmdump(): parser = configargparse.ArgumentParser( usage="%(prog)s [options] [filter]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmdump.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( "--keepserving", - action= "store_true", dest="keepserving", default=False, - help= """ + action="store_true", dest="keepserving", default=False, + help=""" Continue serving after client playback or file read. We exit by default. """ @@ -658,13 +707,13 @@ def mitmdump(): def mitmweb(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmweb.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) group = parser.add_argument_group("Mitmweb") diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 8605d7a14..a2b807bad 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -860,9 +860,9 @@ class FlowMaster(controller.Master): """ if self.server and self.server.config.mode == "reverse": - f.request.host, f.request.port = self.server.config.mode.dst[2:] - f.request.scheme = "https" if self.server.config.mode.dst[ - 1] else "http" + f.request.host = self.server.config.upstream_server.address.host + f.request.port = self.server.config.upstream_server.address.port + f.request.scheme = re.sub("^https?2", "", self.server.config.upstream_server.scheme) f.reply = controller.DummyReply() if f.request: diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4472cb2a0..56d7d57f7 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -10,6 +10,7 @@ from email.utils import parsedate_tz, formatdate, mktime_tz import netlib from netlib import http, tcp, odict, utils, encoding from netlib.http import cookies, http1, http2 +from netlib.http.http1 import HTTP1Protocol from netlib.http.semantics import CONTENT_MISSING from .tcp import TCPHandler @@ -757,7 +758,6 @@ class RequestReplayThread(threading.Thread): server.send(self.flow.server_conn.protocol.assemble(r)) self.flow.server_conn = server - self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) self.flow.response = HTTPResponse.from_protocol( self.flow.server_conn.protocol, r.method, diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index d5dafaaef..61b9a77eb 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -3,8 +3,11 @@ from .root_context import RootContext from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy from .http_proxy import HttpProxy, HttpUpstreamProxy -from .rawtcp import RawTcpLayer +from .transparent_proxy import TransparentProxy +from .http import make_error_response __all__ = [ - "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" + "RootContext", + "Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy", + "make_error_response" ] diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index 9d5a4bebc..76163c71d 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -12,10 +12,7 @@ class ReverseProxy(Layer, ServerConnectionMixin): self._server_tls = server_tls def __call__(self): - if self._client_tls or self._server_tls: - layer = TlsLayer(self, self._client_tls, self._server_tls) - else: - layer = self.ctx.next_layer(self) + layer = TlsLayer(self, self._client_tls, self._server_tls) try: layer() diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 78d484535..af0e7a379 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -4,7 +4,7 @@ from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol from .rawtcp import RawTcpLayer -from .tls import TlsLayer +from .tls import TlsLayer, is_tls_record_magic from .http import Http1Layer, Http2Layer @@ -38,13 +38,7 @@ class RootContext(object): # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello d = top_layer.client_conn.rfile.peek(3) - is_tls_client_hello = ( - len(d) == 3 and - d[0] == '\x16' and - d[1] == '\x03' and - d[2] in ('\x00', '\x01', '\x02', '\x03') - ) - if is_tls_client_hello: + if is_tls_record_magic(d): return TlsLayer(top_layer, True, True) # 3. Check for --tcp diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index 18b363d5f..91935d245 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -8,7 +8,7 @@ from .layer import Layer, ServerConnectionMixin class Socks5Proxy(Layer, ServerConnectionMixin): def __call__(self): try: - s5mode = Socks5ProxyMode(self.config.ssl_ports) + s5mode = Socks5ProxyMode([]) address = s5mode.get_upstream_server(self.client_conn)[2:] except ProxyError as e: # TODO: Unmonkeypatch diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index b1b80034f..850bf5dc7 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -11,6 +11,21 @@ from ..exceptions import ProtocolException from .layer import Layer +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + return ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) + + class TlsLayer(Layer): def __init__(self, ctx, client_tls, server_tls): self.client_sni = None @@ -69,9 +84,13 @@ class TlsLayer(Layer): client_hello_size = 1 offset = 0 while len(client_hello) < client_hello_size: - record_header = self.client_conn.rfile.peek(offset+5)[offset:] + record_header = self.client_conn.rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) != 5: + raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) record_size = struct.unpack("!H", record_header[3:])[0] + 5 - record_body = self.client_conn.rfile.peek(offset+record_size)[offset+5:] + record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) client_hello += record_body offset += record_size client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 @@ -81,7 +100,12 @@ class TlsLayer(Layer): """ Peek into the connection, read the initial client hello and parse it to obtain ALPN values. """ - raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + try: + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + except ProtocolException as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return + try: client_hello = ClientHello.parse(raw_client_hello) except ConstructError as e: @@ -97,7 +121,10 @@ class TlsLayer(Layer): elif extension.type == 0x10: self.client_alpn_protocols = list(extension.alpn_protocols) - self.log("Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), "debug") + self.log( + "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), + "debug" + ) def connect(self): if not self.server_conn: @@ -226,7 +253,8 @@ class TlsLayer(Layer): host = self.server_conn.address.host sans = set() # Incorporate upstream certificate - if self.server_conn and self.server_conn.tls_established and (not self.config.no_upstream_cert): + if self.server_conn and self.server_conn.tls_established and ( + not self.config.no_upstream_cert): upstream_cert = self.server_conn.cert sans.update(upstream_cert.altnames) if upstream_cert.cn: diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index f438e9c22..8ab5a2164 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +import collections import os import re from OpenSSL import SSL @@ -7,6 +8,7 @@ from netlib import certutils, tcp from netlib.http import authentication from .. import utils, platform +from netlib.tcp import Address CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" @@ -15,8 +17,9 @@ CA_DIR = "~/.mitmproxy" # https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" + class HostMatcher(object): - def __init__(self, patterns=[]): + def __init__(self, patterns=tuple()): self.patterns = list(patterns) self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] @@ -32,6 +35,9 @@ class HostMatcher(object): return bool(self.patterns) +ServerSpec = collections.namedtuple("ServerSpec", "scheme address") + + class ProxyConfig: def __init__( self, @@ -41,19 +47,19 @@ class ProxyConfig: clientcerts=None, no_upstream_cert=False, body_size_limit=None, - mode=None, + mode="regular", upstream_server=None, authenticator=None, - ignore_hosts=[], - tcp_hosts=[], + ignore_hosts=tuple(), + tcp_hosts=tuple(), ciphers_client=None, ciphers_server=None, - certs=[], + certs=tuple(), ssl_version_client="secure", ssl_version_server="secure", ssl_verify_upstream_cert=False, - ssl_upstream_trusted_cadir=None, - ssl_upstream_trusted_ca=None, + ssl_verify_upstream_trusted_cadir=None, + ssl_verify_upstream_trusted_ca=None, ): self.host = host self.port = port @@ -63,7 +69,10 @@ class ProxyConfig: self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit self.mode = mode - self.upstream_server = upstream_server + if upstream_server: + self.upstream_server = ServerSpec(upstream_server[0], Address.wrap(upstream_server[1])) + else: + self.upstream_server = None self.check_ignore = HostMatcher(ignore_hosts) self.check_tcp = HostMatcher(tcp_hosts) @@ -76,57 +85,46 @@ class ProxyConfig: for spec, cert in certs: self.certstore.add_cert_file(spec, cert) - self.openssl_method_client, self.openssl_options_client = version_to_openssl( - ssl_version_client) - self.openssl_method_server, self.openssl_options_server = version_to_openssl( - ssl_version_server) + self.openssl_method_client, self.openssl_options_client = \ + sslversion_choices[ssl_version_client] + self.openssl_method_server, self.openssl_options_server = \ + sslversion_choices[ssl_version_server] if ssl_verify_upstream_cert: self.openssl_verification_mode_server = SSL.VERIFY_PEER else: self.openssl_verification_mode_server = SSL.VERIFY_NONE - self.openssl_trusted_cadir_server = ssl_upstream_trusted_cadir - self.openssl_trusted_ca_server = ssl_upstream_trusted_ca + self.openssl_trusted_cadir_server = ssl_verify_upstream_trusted_cadir + self.openssl_trusted_ca_server = ssl_verify_upstream_trusted_ca -sslversion_choices = ( - "all", - "secure", - "SSLv2", - "SSLv3", - "TLSv1", - "TLSv1_1", - "TLSv1_2") - - -def version_to_openssl(version): - """ - Convert a reasonable SSL version specification into the format OpenSSL expects. - Don't ask... - https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 - """ - if version == "all": - return SSL.SSLv23_METHOD, None - elif version == "secure": - # SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+ - # TLSv1_METHOD would be TLS 1.0 only - return SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) - elif version in sslversion_choices: - return getattr(SSL, "%s_METHOD" % version), None - else: - raise ValueError("Invalid SSL version: %s" % version) +""" +Map a reasonable SSL version specification into the format OpenSSL expects. +Don't ask... +https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 +""" +sslversion_choices = { + "all": (SSL.SSLv23_METHOD, 0), + # SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+ + # TLSv1_METHOD would be TLS 1.0 only + "secure": (SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)), + "SSLv2": (SSL.SSLv2_METHOD, 0), + "SSLv3": (SSL.SSLv3_METHOD, 0), + "TLSv1": (SSL.TLSv1_METHOD, 0), + "TLSv1_1": (SSL.TLSv1_1_METHOD, 0), + "TLSv1_2": (SSL.TLSv1_2_METHOD, 0), +} def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) c = 0 - mode, upstream_server, spoofed_ssl_port = None, None, None + mode, upstream_server = "regular", None if options.transparent_proxy: c += 1 if not platform.resolver: - return parser.error( - "Transparent mode not supported on this platform.") + return parser.error("Transparent mode not supported on this platform.") mode = "transparent" if options.socks_proxy: c += 1 @@ -139,32 +137,26 @@ def process_proxy_options(parser, options): c += 1 mode = "upstream" upstream_server = options.upstream_proxy - if options.spoof_mode: - c += 1 - mode = "spoof" - if options.ssl_spoof_mode: - c += 1 - mode = "sslspoof" - spoofed_ssl_port = options.spoofed_ssl_port if c > 1: return parser.error( "Transparent, SOCKS5, reverse and upstream proxy mode " - "are mutually exclusive.") + "are mutually exclusive. Read the docs on proxy modes to understand why." + ) if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) - if not os.path.exists( - options.clientcerts) or not os.path.isdir( - options.clientcerts): + if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): return parser.error( "Client certificate directory does not exist or is not a directory: %s" % - options.clientcerts) + options.clientcerts + ) - if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): + if options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd: if options.auth_singleuser: if len(options.auth_singleuser.split(':')) != 2: return parser.error( - "Invalid single-user specification. Please use the format username:password") + "Invalid single-user specification. Please use the format username:password" + ) username, password = options.auth_singleuser.split(':') password_manager = authentication.PassManSingleUser(username, password) elif options.auth_nonanonymous: @@ -189,12 +181,6 @@ def process_proxy_options(parser, options): parser.error("Certificate file does not exist: %s" % parts[1]) certs.append(parts) - ssl_ports = options.ssl_ports - if options.ssl_ports != TRANSPARENT_SSL_PORTS: - # arparse appends to default value by default, strip that off. - # see http://bugs.python.org/issue16399 - ssl_ports = ssl_ports[len(TRANSPARENT_SSL_PORTS):] - return ProxyConfig( host=options.addr, port=options.port, @@ -204,87 +190,15 @@ def process_proxy_options(parser, options): body_size_limit=body_size_limit, mode=mode, upstream_server=upstream_server, - http_form_in=options.http_form_in, - http_form_out=options.http_form_out, ignore_hosts=options.ignore_hosts, tcp_hosts=options.tcp_hosts, authenticator=authenticator, ciphers_client=options.ciphers_client, ciphers_server=options.ciphers_server, - certs=certs, + certs=tuple(certs), ssl_version_client=options.ssl_version_client, ssl_version_server=options.ssl_version_server, - ssl_ports=ssl_ports, - spoofed_ssl_port=spoofed_ssl_port, ssl_verify_upstream_cert=options.ssl_verify_upstream_cert, - ssl_upstream_trusted_cadir=options.ssl_upstream_trusted_cadir, - ssl_upstream_trusted_ca=options.ssl_upstream_trusted_ca - ) - - -def ssl_option_group(parser): - group = parser.add_argument_group("SSL") - group.add_argument( - "--cert", - dest='certs', - default=[], - type=str, - metavar="SPEC", - action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' - 'The domain may include a wildcard, and is equal to "*" if not specified. ' - 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' - 'it is used, else the default key in the conf dir is used. ' - 'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. ' - 'Can be passed multiple times.') - group.add_argument( - "--ciphers-client", action="store", - type=str, dest="ciphers_client", default=DEFAULT_CLIENT_CIPHERS, - help="Set supported ciphers for client connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--ciphers-server", action="store", - type=str, dest="ciphers_server", default=None, - help="Set supported ciphers for server connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--client-certs", action="store", - type=str, dest="clientcerts", default=None, - help="Client certificate directory." - ) - group.add_argument( - "--no-upstream-cert", default=False, - action="store_true", dest="no_upstream_cert", - help="Don't connect to upstream server to look up certificate details." - ) - group.add_argument( - "--verify-upstream-cert", default=False, - action="store_true", dest="ssl_verify_upstream_cert", - help="Verify upstream server SSL/TLS certificates and fail if invalid " - "or not present." - ) - group.add_argument( - "--upstream-trusted-cadir", default=None, action="store", - dest="ssl_upstream_trusted_cadir", - help="Path to a directory of trusted CA certificates for upstream " - "server verification prepared using the c_rehash tool." - ) - group.add_argument( - "--upstream-trusted-ca", default=None, action="store", - dest="ssl_upstream_trusted_ca", - help="Path to a PEM formatted trusted CA certificate." - ) - group.add_argument( - "--ssl-version-client", dest="ssl_version_client", - default="secure", action="store", - choices=sslversion_choices, - help="Set supported SSL/TLS version for client connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." - ) - group.add_argument( - "--ssl-version-server", dest="ssl_version_server", - default="secure", action="store", - choices=sslversion_choices, - help="Set supported SSL/TLS version for server connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." - ) + ssl_verify_upstream_trusted_cadir=options.ssl_verify_upstream_trusted_cadir, + ssl_verify_upstream_trusted_ca=options.ssl_verify_upstream_trusted_ca + ) \ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 19ddb9301..1fc4cbdaf 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -5,6 +5,8 @@ import sys import socket from libmproxy.protocol2.layer import Kill from netlib import tcp +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError from ..protocol.handle import protocol_handler from .. import protocol2 @@ -82,11 +84,31 @@ class ConnectionHandler2: self.channel ) - # FIXME: properly parse config - if self.config.mode == "upstream": - root_layer = protocol2.HttpUpstreamProxy(root_context, ("localhost", 8081)) - else: + mode = self.config.mode + if mode == "upstream": + root_layer = protocol2.HttpUpstreamProxy( + root_context, + self.config.upstream_server.address + ) + elif mode == "transparent": + root_layer = protocol2.TransparentProxy(root_context) + elif mode == "reverse": + client_tls = self.config.upstream_server.scheme.startswith("https") + server_tls = self.config.upstream_server.scheme.endswith("https") + root_layer = protocol2.ReverseProxy( + root_context, + self.config.upstream_server.address, + client_tls, + server_tls + ) + elif mode == "socks5": + root_layer = protocol2.Socks5Proxy(root_context) + elif mode == "regular": root_layer = protocol2.HttpProxy(root_context) + elif callable(mode): # pragma: nocover + root_layer = mode(root_context) + else: # pragma: nocover + raise ValueError("Unknown proxy mode: %s" % mode) try: root_layer() @@ -94,6 +116,14 @@ class ConnectionHandler2: self.log("Connection killed", "info") except ProtocolException as e: self.log(e, "info") + # If an error propagates to the topmost level, + # we send an HTTP error response, which is both + # understandable by HTTP clients and humans. + try: + error_response = protocol2.make_error_response(502, repr(e)) + self.client_conn.send(HTTP1Protocol().assemble(error_response)) + except NetLibError: + pass except Exception: self.log(traceback.format_exc(), "error") print(traceback.format_exc(), file=sys.stderr) diff --git a/test/test_cmdline.py b/test/test_cmdline.py index eafcbde48..ee2f7044b 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -38,15 +38,15 @@ def test_parse_replace_hook(): def test_parse_server_spec(): tutils.raises("Invalid server specification", cmdline.parse_server_spec, "") assert cmdline.parse_server_spec( - "http://foo.com:88") == [False, False, "foo.com", 88] + "http://foo.com:88") == ("http", ("foo.com", 88)) assert cmdline.parse_server_spec( - "http://foo.com") == [False, False, "foo.com", 80] + "http://foo.com") == ("http", ("foo.com", 80)) assert cmdline.parse_server_spec( - "https://foo.com") == [True, True, "foo.com", 443] + "https://foo.com") == ("https", ("foo.com", 443)) assert cmdline.parse_server_spec_special( - "https2http://foo.com") == [True, False, "foo.com", 80] + "https2http://foo.com") == ("https2http", ("foo.com", 80)) assert cmdline.parse_server_spec_special( - "http2https://foo.com") == [False, True, "foo.com", 443] + "http2https://foo.com") == ("http2https", ("foo.com", 443)) tutils.raises( "Invalid server specification", cmdline.parse_server_spec, @@ -55,6 +55,10 @@ def test_parse_server_spec(): "Invalid server specification", cmdline.parse_server_spec, "http://") + tutils.raises( + "Invalid server specification", + cmdline.parse_server_spec, + "https2http://foo.com") def test_parse_setheaders(): diff --git a/test/test_flow.py b/test/test_flow.py index 711688da0..5c49deed3 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -4,6 +4,7 @@ import os.path from cStringIO import StringIO import email.utils import mock +from libmproxy.cmdline import parse_server_spec import netlib.utils from netlib import odict @@ -672,11 +673,8 @@ class TestSerialize: s = flow.State() conf = ProxyConfig( mode="reverse", - upstream_server=[ - True, - True, - "use-this-domain", - 80]) + upstream_server=("https", ("use-this-domain", 80)) + ) fm = flow.FlowMaster(DummyServer(conf), s) fm.load_flows(r) assert s.flows[0].request.host == "use-this-domain" diff --git a/test/test_proxy.py b/test/test_proxy.py index 6ab19e02f..9c01ab639 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -97,13 +97,7 @@ class TestProcessProxyOptions: self.assert_err("expected one argument", "-U") self.assert_err("Invalid server specification", "-U", "upstream") - self.assert_noerr("--spoof") - self.assert_noerr("--ssl-spoof") - - self.assert_noerr("--spoofed-port", "443") - self.assert_err("expected one argument", "--spoofed-port") - - self.assert_err("mutually exclusive", "-R", "http://localhost", "-T") + self.assert_err("not allowed with", "-R", "http://localhost", "-T") def test_client_certs(self): with tutils.tmpdir() as cadir: diff --git a/test/test_server.py b/test/test_server.py index e9c40d1a0..1216a349a 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -464,60 +464,11 @@ class TestSocks5(tservers.SocksModeTest): assert "SOCKS5 mode failure" in f.content -class TestSpoof(tservers.SpoofModeTest): - def test_http(self): - alist = ( - ("localhost", self.server.port), - ("127.0.0.1", self.server.port) - ) - for a in alist: - self.server.clear_log() - p = self.pathoc() - f = p.request("get:/p/304:h'Host'='%s:%s'" % a) - assert self.server.last_log() - assert f.status_code == 304 - l = self.master.state.view[-1] - assert l.server_conn.address - assert l.server_conn.address.host == a[0] - assert l.server_conn.address.port == a[1] - - def test_http_without_host(self): - p = self.pathoc() - f = p.request("get:/p/304:r") - assert f.status_code == 400 - - -class TestSSLSpoof(tservers.SSLSpoofModeTest): - def test_https(self): - alist = ( - ("localhost", self.server.port), - ("127.0.0.1", self.server.port) - ) - for a in alist: - self.server.clear_log() - self.config.mode.sslport = a[1] - p = self.pathoc(sni=a[0]) - f = p.request("get:/p/304") - assert self.server.last_log() - assert f.status_code == 304 - l = self.master.state.view[-1] - assert l.server_conn.address - assert l.server_conn.address.host == a[0] - assert l.server_conn.address.port == a[1] - - def test_https_without_sni(self): - a = ("localhost", self.server.port) - self.config.mode.sslport = a[1] - p = self.pathoc(sni=None) - f = p.request("get:/p/304") - assert f.status_code == 400 - - class TestHttps2Http(tservers.ReverseProxTest): @classmethod def get_proxy_config(cls): d = super(TestHttps2Http, cls).get_proxy_config() - d["upstream_server"][0] = True + d["upstream_server"] = ("https2http", d["upstream_server"][1]) return d def pathoc(self, ssl, sni=None): @@ -541,7 +492,7 @@ class TestHttps2Http(tservers.ReverseProxTest): def test_http(self): p = self.pathoc(ssl=False) - assert p.request("get:'/p/200'").status_code == 400 + assert p.request("get:'/p/200'").status_code == 502 class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin): diff --git a/test/tservers.py b/test/tservers.py index 3c73b2621..43ebf2bb2 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -1,6 +1,5 @@ import os.path import threading -import Queue import shutil import tempfile import flask @@ -130,7 +129,6 @@ class ProxTestBase(object): no_upstream_cert = cls.no_upstream_cert, cadir = cls.cadir, authenticator = cls.authenticator, - ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []), clientcerts = tutils.test_data.path("data/clientcert") if cls.clientcerts else None ) @@ -235,12 +233,10 @@ class ReverseProxTest(ProxTestBase): @classmethod def get_proxy_config(cls): d = ProxTestBase.get_proxy_config() - d["upstream_server"] = [ - True if cls.ssl else False, - True if cls.ssl else False, - "127.0.0.1", - cls.server.port - ] + d["upstream_server"] = ( + "https" if cls.ssl else "http", + ("127.0.0.1", cls.server.port) + ) d["mode"] = "reverse" return d @@ -360,7 +356,7 @@ class ChainProxTest(ProxTestBase): if cls.chain: # First proxy is in normal mode. d.update( mode="upstream", - upstream_server=(False, False, "127.0.0.1", cls.chain[0].port) + upstream_server=("http", ("127.0.0.1", cls.chain[0].port)) ) return d