clean up config/cmdline, fix bugs, remove cruft

This commit is contained in:
Maximilian Hils 2015-08-28 01:51:13 +02:00
parent a86491eeed
commit 1cc48345e1
15 changed files with 304 additions and 346 deletions

View File

@ -2,8 +2,8 @@ from __future__ import absolute_import
import os
import re
import configargparse
from netlib.tcp import Address
from netlib import http
import netlib.utils
from . import filt, utils, version
@ -102,32 +102,22 @@ def parse_setheader(s):
return _parse_hook(s)
def parse_server_spec(url):
def parse_server_spec(url, allowed_schemes=("http", "https")):
p = netlib.utils.parse_url(url)
if not p or not p[1] or p[0] not in ("http", "https"):
if not p or not p[1] or p[0] not in allowed_schemes:
raise configargparse.ArgumentTypeError(
"Invalid server specification: %s" % url
)
if p[0].lower() == "https":
ssl = [True, True]
else:
ssl = [False, False]
return ssl + list(p[1:3])
address = Address(p[1:3])
scheme = p[0].lower()
return config.ServerSpec(scheme, address)
def parse_server_spec_special(url):
"""
Provides additional support for http2https and https2http schemes.
"""
normalized_url = re.sub("^https?2", "", url)
ret = parse_server_spec(normalized_url)
if url.lower().startswith("https2http"):
ret[0] = True
elif url.lower().startswith("http2https"):
ret[0] = False
return ret
return parse_server_spec(url, allowed_schemes=("http", "https", "http2https", "https2http"))
def get_common_options(options):
@ -192,24 +182,24 @@ def get_common_options(options):
outfile=options.outfile,
verbosity=options.verbose,
nopop=options.nopop,
replay_ignore_content = options.replay_ignore_content,
replay_ignore_params = options.replay_ignore_params,
replay_ignore_payload_params = options.replay_ignore_payload_params,
replay_ignore_host = options.replay_ignore_host
replay_ignore_content=options.replay_ignore_content,
replay_ignore_params=options.replay_ignore_params,
replay_ignore_payload_params=options.replay_ignore_payload_params,
replay_ignore_host=options.replay_ignore_host
)
def common_options(parser):
def basic_options(parser):
parser.add_argument(
'--version',
action= 'version',
version= "%(prog)s" + " " + version.VERSION
action='version',
version="%(prog)s" + " " + version.VERSION
)
parser.add_argument(
'--shortversion',
action= 'version',
help = "show program's short version number and exit",
version = version.VERSION
action='version',
help="show program's short version number and exit",
version=version.VERSION
)
parser.add_argument(
"--anticache",
@ -301,11 +291,42 @@ def common_options(parser):
"""
)
def proxy_modes(parser):
group = parser.add_argument_group("Proxy Modes").add_mutually_exclusive_group()
group.add_argument(
"-R", "--reverse",
action="store",
type=parse_server_spec_special,
dest="reverse_proxy",
default=None,
help="""
Forward all requests to upstream HTTP server:
http[s][2http[s]]://host[:port]
"""
)
group.add_argument(
"--socks",
action="store_true", dest="socks_proxy", default=False,
help="Set SOCKS5 proxy mode."
)
group.add_argument(
"-T", "--transparent",
action="store_true", dest="transparent_proxy", default=False,
help="Set transparent proxy mode."
)
group.add_argument(
"-U", "--upstream",
action="store",
type=parse_server_spec,
dest="upstream_proxy",
default=None,
help="Forward all requests to upstream proxy server: http://host[:port]"
)
def proxy_options(parser):
group = parser.add_argument_group("Proxy Options")
# We could make a mutually exclusive group out of -R, -U, -T, but we don't
# do that because - --upstream-server should be in that group as well, but
# it's already in a different group. - our own error messages are more
# helpful
group.add_argument(
"-b", "--bind-address",
action="store", type=str, dest="addr", default='',
@ -344,70 +365,78 @@ def common_options(parser):
action="store", type=int, dest="port", default=8080,
help="Proxy service port."
)
def proxy_ssl_options(parser):
# TODO: Agree to consistently either use "upstream" or "server".
group = parser.add_argument_group("SSL")
group.add_argument(
"-R", "--reverse",
action="store",
type=parse_server_spec_special,
dest="reverse_proxy",
default=None,
help="""
Forward all requests to upstream HTTP server:
http[s][2http[s]]://host[:port]
"""
"--cert",
dest='certs',
default=[],
type=str,
metavar="SPEC",
action="append",
help='Add an SSL certificate. SPEC is of the form "[domain=]path". '
'The domain may include a wildcard, and is equal to "*" if not specified. '
'The file at path is a certificate in PEM format. If a private key is included '
'in the PEM, it is used, else the default key in the conf dir is used. '
'The PEM file should contain the full certificate chain, with the leaf certificate '
'as the first entry. Can be passed multiple times.')
group.add_argument(
"--ciphers-client", action="store",
type=str, dest="ciphers_client", default=config.DEFAULT_CLIENT_CIPHERS,
help="Set supported ciphers for client connections. (OpenSSL Syntax)"
)
group.add_argument(
"--socks",
action="store_true", dest="socks_proxy", default=False,
help="Set SOCKS5 proxy mode."
"--ciphers-server", action="store",
type=str, dest="ciphers_server", default=None,
help="Set supported ciphers for server connections. (OpenSSL Syntax)"
)
group.add_argument(
"-T", "--transparent",
action="store_true", dest="transparent_proxy", default=False,
help="Set transparent proxy mode."
"--client-certs", action="store",
type=str, dest="clientcerts", default=None,
help="Client certificate directory."
)
group.add_argument(
"-U", "--upstream",
action="store",
type=parse_server_spec,
dest="upstream_proxy",
default=None,
help="Forward all requests to upstream proxy server: http://host[:port]"
"--no-upstream-cert", default=False,
action="store_true", dest="no_upstream_cert",
help="Don't connect to upstream server to look up certificate details."
)
group.add_argument(
"--spoof",
action="store_true", dest="spoof_mode", default=False,
help="Use Host header to connect to HTTP servers."
"--verify-upstream-cert", default=False,
action="store_true", dest="ssl_verify_upstream_cert",
help="Verify upstream server SSL/TLS certificates and fail if invalid "
"or not present."
)
group.add_argument(
"--ssl-spoof",
action="store_true", dest="ssl_spoof_mode", default=False,
help="Use TLS SNI to connect to HTTPS servers."
"--upstream-trusted-cadir", default=None, action="store",
dest="ssl_verify_upstream_trusted_cadir",
help="Path to a directory of trusted CA certificates for upstream "
"server verification prepared using the c_rehash tool."
)
group.add_argument(
"--spoofed-port",
action="store", dest="spoofed_ssl_port", type=int, default=443,
help="Port number of upstream HTTPS servers in SSL spoof mode."
"--upstream-trusted-ca", default=None, action="store",
dest="ssl_verify_upstream_trusted_ca",
help="Path to a PEM formatted trusted CA certificate."
)
group.add_argument(
"--ssl-version-client", dest="ssl_version_client",
default="secure", action="store",
choices=config.sslversion_choices.keys(),
help="Set supported SSL/TLS version for client connections. "
"SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
)
group.add_argument(
"--ssl-version-server", dest="ssl_version_server",
default="secure", action="store",
choices=config.sslversion_choices.keys(),
help="Set supported SSL/TLS version for server connections. "
"SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
)
group = parser.add_argument_group(
"Advanced Proxy Options",
"""
The following options allow a custom adjustment of the proxy
behavior. Normally, you don't want to use these options directly and
use the provided wrappers instead (-R, -U, -T).
"""
)
group.add_argument(
"--http-form-in", dest="http_form_in", default=None,
action="store", choices=("relative", "absolute"),
help="Override the HTTP request form accepted by the proxy"
)
group.add_argument(
"--http-form-out", dest="http_form_out", default=None,
action="store", choices=("relative", "absolute"),
help="Override the HTTP request form sent upstream by the proxy"
)
def onboarding_app(parser):
group = parser.add_argument_group("Onboarding App")
group.add_argument(
"--noapp",
@ -433,6 +462,8 @@ def common_options(parser):
help="Port to serve the onboarding app from."
)
def client_replay(parser):
group = parser.add_argument_group("Client Replay")
group.add_argument(
"-c", "--client-replay",
@ -440,6 +471,8 @@ def common_options(parser):
help="Replay client requests from a saved file."
)
def server_replay(parser):
group = parser.add_argument_group("Server Replay")
group.add_argument(
"-S", "--server-replay",
@ -504,6 +537,8 @@ def common_options(parser):
default=False,
help="Ignore request's destination host while searching for a saved flow to replay")
def replacements(parser):
group = parser.add_argument_group(
"Replacements",
"""
@ -520,14 +555,16 @@ def common_options(parser):
)
group.add_argument(
"--replace-from-file",
action = "append", type=str, dest="replace_file", default=[],
metavar = "PATH",
help = """
action="append", type=str, dest="replace_file", default=[],
metavar="PATH",
help="""
Replacement pattern, where the replacement clause is a path to a
file.
"""
)
def set_headers(parser):
group = parser.add_argument_group(
"Set Headers",
"""
@ -543,21 +580,22 @@ def common_options(parser):
help="Header set pattern."
)
def proxy_authentication(parser):
group = parser.add_argument_group(
"Proxy Authentication",
"""
Specify which users are allowed to access the proxy and the method
used for authenticating them.
"""
)
user_specification_group = group.add_mutually_exclusive_group()
user_specification_group.add_argument(
).add_mutually_exclusive_group()
group.add_argument(
"--nonanonymous",
action="store_true", dest="auth_nonanonymous",
help="Allow access to any user long as a credentials are specified."
)
user_specification_group.add_argument(
group.add_argument(
"--singleuser",
action="store", dest="auth_singleuser", type=str,
metavar="USER",
@ -566,14 +604,25 @@ def common_options(parser):
username:password.
"""
)
user_specification_group.add_argument(
group.add_argument(
"--htpasswd",
action="store", dest="auth_htpasswd", type=str,
metavar="PATH",
help="Allow access to users specified in an Apache htpasswd file."
)
config.ssl_option_group(parser)
def common_options(parser):
basic_options(parser)
proxy_modes(parser)
proxy_options(parser)
proxy_ssl_options(parser)
onboarding_app(parser)
client_replay(parser)
server_replay(parser)
replacements(parser)
set_headers(parser)
proxy_authentication(parser)
def mitmproxy():
@ -583,13 +632,13 @@ def mitmproxy():
parser = configargparse.ArgumentParser(
usage="%(prog)s [options]",
args_for_setting_config_path = ["--conf"],
default_config_files = [
args_for_setting_config_path=["--conf"],
default_config_files=[
os.path.join(config.CA_DIR, "common.conf"),
os.path.join(config.CA_DIR, "mitmproxy.conf")
],
add_config_file_help = True,
add_env_var_help = True
add_config_file_help=True,
add_env_var_help=True
)
common_options(parser)
parser.add_argument(
@ -628,20 +677,20 @@ def mitmproxy():
def mitmdump():
parser = configargparse.ArgumentParser(
usage="%(prog)s [options] [filter]",
args_for_setting_config_path = ["--conf"],
default_config_files = [
args_for_setting_config_path=["--conf"],
default_config_files=[
os.path.join(config.CA_DIR, "common.conf"),
os.path.join(config.CA_DIR, "mitmdump.conf")
],
add_config_file_help = True,
add_env_var_help = True
add_config_file_help=True,
add_env_var_help=True
)
common_options(parser)
parser.add_argument(
"--keepserving",
action= "store_true", dest="keepserving", default=False,
help= """
action="store_true", dest="keepserving", default=False,
help="""
Continue serving after client playback or file read. We exit by
default.
"""
@ -658,13 +707,13 @@ def mitmdump():
def mitmweb():
parser = configargparse.ArgumentParser(
usage="%(prog)s [options]",
args_for_setting_config_path = ["--conf"],
default_config_files = [
args_for_setting_config_path=["--conf"],
default_config_files=[
os.path.join(config.CA_DIR, "common.conf"),
os.path.join(config.CA_DIR, "mitmweb.conf")
],
add_config_file_help = True,
add_env_var_help = True
add_config_file_help=True,
add_env_var_help=True
)
group = parser.add_argument_group("Mitmweb")

View File

@ -860,9 +860,9 @@ class FlowMaster(controller.Master):
"""
if self.server and self.server.config.mode == "reverse":
f.request.host, f.request.port = self.server.config.mode.dst[2:]
f.request.scheme = "https" if self.server.config.mode.dst[
1] else "http"
f.request.host = self.server.config.upstream_server.address.host
f.request.port = self.server.config.upstream_server.address.port
f.request.scheme = re.sub("^https?2", "", self.server.config.upstream_server.scheme)
f.reply = controller.DummyReply()
if f.request:

View File

@ -10,6 +10,7 @@ from email.utils import parsedate_tz, formatdate, mktime_tz
import netlib
from netlib import http, tcp, odict, utils, encoding
from netlib.http import cookies, http1, http2
from netlib.http.http1 import HTTP1Protocol
from netlib.http.semantics import CONTENT_MISSING
from .tcp import TCPHandler
@ -757,7 +758,6 @@ class RequestReplayThread(threading.Thread):
server.send(self.flow.server_conn.protocol.assemble(r))
self.flow.server_conn = server
self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn)
self.flow.response = HTTPResponse.from_protocol(
self.flow.server_conn.protocol,
r.method,

View File

@ -3,8 +3,11 @@ from .root_context import RootContext
from .socks_proxy import Socks5Proxy
from .reverse_proxy import ReverseProxy
from .http_proxy import HttpProxy, HttpUpstreamProxy
from .rawtcp import RawTcpLayer
from .transparent_proxy import TransparentProxy
from .http import make_error_response
__all__ = [
"Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy"
"RootContext",
"Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy",
"make_error_response"
]

View File

@ -12,10 +12,7 @@ class ReverseProxy(Layer, ServerConnectionMixin):
self._server_tls = server_tls
def __call__(self):
if self._client_tls or self._server_tls:
layer = TlsLayer(self, self._client_tls, self._server_tls)
else:
layer = self.ctx.next_layer(self)
layer = TlsLayer(self, self._client_tls, self._server_tls)
try:
layer()

View File

@ -4,7 +4,7 @@ from netlib.http.http1 import HTTP1Protocol
from netlib.http.http2 import HTTP2Protocol
from .rawtcp import RawTcpLayer
from .tls import TlsLayer
from .tls import TlsLayer, is_tls_record_magic
from .http import Http1Layer, Http2Layer
@ -38,13 +38,7 @@ class RootContext(object):
# TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2
# http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello
d = top_layer.client_conn.rfile.peek(3)
is_tls_client_hello = (
len(d) == 3 and
d[0] == '\x16' and
d[1] == '\x03' and
d[2] in ('\x00', '\x01', '\x02', '\x03')
)
if is_tls_client_hello:
if is_tls_record_magic(d):
return TlsLayer(top_layer, True, True)
# 3. Check for --tcp

View File

@ -8,7 +8,7 @@ from .layer import Layer, ServerConnectionMixin
class Socks5Proxy(Layer, ServerConnectionMixin):
def __call__(self):
try:
s5mode = Socks5ProxyMode(self.config.ssl_ports)
s5mode = Socks5ProxyMode([])
address = s5mode.get_upstream_server(self.client_conn)[2:]
except ProxyError as e:
# TODO: Unmonkeypatch

View File

@ -11,6 +11,21 @@ from ..exceptions import ProtocolException
from .layer import Layer
def is_tls_record_magic(d):
"""
Returns:
True, if the passed bytes start with the TLS record magic bytes.
False, otherwise.
"""
d = d[:3]
return (
len(d) == 3 and
d[0] == '\x16' and
d[1] == '\x03' and
d[2] in ('\x00', '\x01', '\x02', '\x03')
)
class TlsLayer(Layer):
def __init__(self, ctx, client_tls, server_tls):
self.client_sni = None
@ -69,9 +84,13 @@ class TlsLayer(Layer):
client_hello_size = 1
offset = 0
while len(client_hello) < client_hello_size:
record_header = self.client_conn.rfile.peek(offset+5)[offset:]
record_header = self.client_conn.rfile.peek(offset + 5)[offset:]
if not is_tls_record_magic(record_header) or len(record_header) != 5:
raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header)
record_size = struct.unpack("!H", record_header[3:])[0] + 5
record_body = self.client_conn.rfile.peek(offset+record_size)[offset+5:]
record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:]
if len(record_body) != record_size - 5:
raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body)
client_hello += record_body
offset += record_size
client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4
@ -81,7 +100,12 @@ class TlsLayer(Layer):
"""
Peek into the connection, read the initial client hello and parse it to obtain ALPN values.
"""
raw_client_hello = self._get_client_hello()[4:] # exclude handshake header.
try:
raw_client_hello = self._get_client_hello()[4:] # exclude handshake header.
except ProtocolException as e:
self.log("Cannot parse Client Hello: %s" % repr(e), "error")
return
try:
client_hello = ClientHello.parse(raw_client_hello)
except ConstructError as e:
@ -97,7 +121,10 @@ class TlsLayer(Layer):
elif extension.type == 0x10:
self.client_alpn_protocols = list(extension.alpn_protocols)
self.log("Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), "debug")
self.log(
"Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols),
"debug"
)
def connect(self):
if not self.server_conn:
@ -226,7 +253,8 @@ class TlsLayer(Layer):
host = self.server_conn.address.host
sans = set()
# Incorporate upstream certificate
if self.server_conn and self.server_conn.tls_established and (not self.config.no_upstream_cert):
if self.server_conn and self.server_conn.tls_established and (
not self.config.no_upstream_cert):
upstream_cert = self.server_conn.cert
sans.update(upstream_cert.altnames)
if upstream_cert.cn:

View File

@ -1,4 +1,5 @@
from __future__ import absolute_import
import collections
import os
import re
from OpenSSL import SSL
@ -7,6 +8,7 @@ from netlib import certutils, tcp
from netlib.http import authentication
from .. import utils, platform
from netlib.tcp import Address
CONF_BASENAME = "mitmproxy"
CA_DIR = "~/.mitmproxy"
@ -15,8 +17,9 @@ CA_DIR = "~/.mitmproxy"
# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old
DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA"
class HostMatcher(object):
def __init__(self, patterns=[]):
def __init__(self, patterns=tuple()):
self.patterns = list(patterns)
self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns]
@ -32,6 +35,9 @@ class HostMatcher(object):
return bool(self.patterns)
ServerSpec = collections.namedtuple("ServerSpec", "scheme address")
class ProxyConfig:
def __init__(
self,
@ -41,19 +47,19 @@ class ProxyConfig:
clientcerts=None,
no_upstream_cert=False,
body_size_limit=None,
mode=None,
mode="regular",
upstream_server=None,
authenticator=None,
ignore_hosts=[],
tcp_hosts=[],
ignore_hosts=tuple(),
tcp_hosts=tuple(),
ciphers_client=None,
ciphers_server=None,
certs=[],
certs=tuple(),
ssl_version_client="secure",
ssl_version_server="secure",
ssl_verify_upstream_cert=False,
ssl_upstream_trusted_cadir=None,
ssl_upstream_trusted_ca=None,
ssl_verify_upstream_trusted_cadir=None,
ssl_verify_upstream_trusted_ca=None,
):
self.host = host
self.port = port
@ -63,7 +69,10 @@ class ProxyConfig:
self.no_upstream_cert = no_upstream_cert
self.body_size_limit = body_size_limit
self.mode = mode
self.upstream_server = upstream_server
if upstream_server:
self.upstream_server = ServerSpec(upstream_server[0], Address.wrap(upstream_server[1]))
else:
self.upstream_server = None
self.check_ignore = HostMatcher(ignore_hosts)
self.check_tcp = HostMatcher(tcp_hosts)
@ -76,57 +85,46 @@ class ProxyConfig:
for spec, cert in certs:
self.certstore.add_cert_file(spec, cert)
self.openssl_method_client, self.openssl_options_client = version_to_openssl(
ssl_version_client)
self.openssl_method_server, self.openssl_options_server = version_to_openssl(
ssl_version_server)
self.openssl_method_client, self.openssl_options_client = \
sslversion_choices[ssl_version_client]
self.openssl_method_server, self.openssl_options_server = \
sslversion_choices[ssl_version_server]
if ssl_verify_upstream_cert:
self.openssl_verification_mode_server = SSL.VERIFY_PEER
else:
self.openssl_verification_mode_server = SSL.VERIFY_NONE
self.openssl_trusted_cadir_server = ssl_upstream_trusted_cadir
self.openssl_trusted_ca_server = ssl_upstream_trusted_ca
self.openssl_trusted_cadir_server = ssl_verify_upstream_trusted_cadir
self.openssl_trusted_ca_server = ssl_verify_upstream_trusted_ca
sslversion_choices = (
"all",
"secure",
"SSLv2",
"SSLv3",
"TLSv1",
"TLSv1_1",
"TLSv1_2")
def version_to_openssl(version):
"""
Convert a reasonable SSL version specification into the format OpenSSL expects.
Don't ask...
https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3
"""
if version == "all":
return SSL.SSLv23_METHOD, None
elif version == "secure":
# SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+
# TLSv1_METHOD would be TLS 1.0 only
return SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)
elif version in sslversion_choices:
return getattr(SSL, "%s_METHOD" % version), None
else:
raise ValueError("Invalid SSL version: %s" % version)
"""
Map a reasonable SSL version specification into the format OpenSSL expects.
Don't ask...
https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3
"""
sslversion_choices = {
"all": (SSL.SSLv23_METHOD, 0),
# SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+
# TLSv1_METHOD would be TLS 1.0 only
"secure": (SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)),
"SSLv2": (SSL.SSLv2_METHOD, 0),
"SSLv3": (SSL.SSLv3_METHOD, 0),
"TLSv1": (SSL.TLSv1_METHOD, 0),
"TLSv1_1": (SSL.TLSv1_1_METHOD, 0),
"TLSv1_2": (SSL.TLSv1_2_METHOD, 0),
}
def process_proxy_options(parser, options):
body_size_limit = utils.parse_size(options.body_size_limit)
c = 0
mode, upstream_server, spoofed_ssl_port = None, None, None
mode, upstream_server = "regular", None
if options.transparent_proxy:
c += 1
if not platform.resolver:
return parser.error(
"Transparent mode not supported on this platform.")
return parser.error("Transparent mode not supported on this platform.")
mode = "transparent"
if options.socks_proxy:
c += 1
@ -139,32 +137,26 @@ def process_proxy_options(parser, options):
c += 1
mode = "upstream"
upstream_server = options.upstream_proxy
if options.spoof_mode:
c += 1
mode = "spoof"
if options.ssl_spoof_mode:
c += 1
mode = "sslspoof"
spoofed_ssl_port = options.spoofed_ssl_port
if c > 1:
return parser.error(
"Transparent, SOCKS5, reverse and upstream proxy mode "
"are mutually exclusive.")
"are mutually exclusive. Read the docs on proxy modes to understand why."
)
if options.clientcerts:
options.clientcerts = os.path.expanduser(options.clientcerts)
if not os.path.exists(
options.clientcerts) or not os.path.isdir(
options.clientcerts):
if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts):
return parser.error(
"Client certificate directory does not exist or is not a directory: %s" %
options.clientcerts)
options.clientcerts
)
if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd):
if options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd:
if options.auth_singleuser:
if len(options.auth_singleuser.split(':')) != 2:
return parser.error(
"Invalid single-user specification. Please use the format username:password")
"Invalid single-user specification. Please use the format username:password"
)
username, password = options.auth_singleuser.split(':')
password_manager = authentication.PassManSingleUser(username, password)
elif options.auth_nonanonymous:
@ -189,12 +181,6 @@ def process_proxy_options(parser, options):
parser.error("Certificate file does not exist: %s" % parts[1])
certs.append(parts)
ssl_ports = options.ssl_ports
if options.ssl_ports != TRANSPARENT_SSL_PORTS:
# arparse appends to default value by default, strip that off.
# see http://bugs.python.org/issue16399
ssl_ports = ssl_ports[len(TRANSPARENT_SSL_PORTS):]
return ProxyConfig(
host=options.addr,
port=options.port,
@ -204,87 +190,15 @@ def process_proxy_options(parser, options):
body_size_limit=body_size_limit,
mode=mode,
upstream_server=upstream_server,
http_form_in=options.http_form_in,
http_form_out=options.http_form_out,
ignore_hosts=options.ignore_hosts,
tcp_hosts=options.tcp_hosts,
authenticator=authenticator,
ciphers_client=options.ciphers_client,
ciphers_server=options.ciphers_server,
certs=certs,
certs=tuple(certs),
ssl_version_client=options.ssl_version_client,
ssl_version_server=options.ssl_version_server,
ssl_ports=ssl_ports,
spoofed_ssl_port=spoofed_ssl_port,
ssl_verify_upstream_cert=options.ssl_verify_upstream_cert,
ssl_upstream_trusted_cadir=options.ssl_upstream_trusted_cadir,
ssl_upstream_trusted_ca=options.ssl_upstream_trusted_ca
)
def ssl_option_group(parser):
group = parser.add_argument_group("SSL")
group.add_argument(
"--cert",
dest='certs',
default=[],
type=str,
metavar="SPEC",
action="append",
help='Add an SSL certificate. SPEC is of the form "[domain=]path". '
'The domain may include a wildcard, and is equal to "*" if not specified. '
'The file at path is a certificate in PEM format. If a private key is included in the PEM, '
'it is used, else the default key in the conf dir is used. '
'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. '
'Can be passed multiple times.')
group.add_argument(
"--ciphers-client", action="store",
type=str, dest="ciphers_client", default=DEFAULT_CLIENT_CIPHERS,
help="Set supported ciphers for client connections. (OpenSSL Syntax)"
)
group.add_argument(
"--ciphers-server", action="store",
type=str, dest="ciphers_server", default=None,
help="Set supported ciphers for server connections. (OpenSSL Syntax)"
)
group.add_argument(
"--client-certs", action="store",
type=str, dest="clientcerts", default=None,
help="Client certificate directory."
)
group.add_argument(
"--no-upstream-cert", default=False,
action="store_true", dest="no_upstream_cert",
help="Don't connect to upstream server to look up certificate details."
)
group.add_argument(
"--verify-upstream-cert", default=False,
action="store_true", dest="ssl_verify_upstream_cert",
help="Verify upstream server SSL/TLS certificates and fail if invalid "
"or not present."
)
group.add_argument(
"--upstream-trusted-cadir", default=None, action="store",
dest="ssl_upstream_trusted_cadir",
help="Path to a directory of trusted CA certificates for upstream "
"server verification prepared using the c_rehash tool."
)
group.add_argument(
"--upstream-trusted-ca", default=None, action="store",
dest="ssl_upstream_trusted_ca",
help="Path to a PEM formatted trusted CA certificate."
)
group.add_argument(
"--ssl-version-client", dest="ssl_version_client",
default="secure", action="store",
choices=sslversion_choices,
help="Set supported SSL/TLS version for client connections. "
"SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure."
)
group.add_argument(
"--ssl-version-server", dest="ssl_version_server",
default="secure", action="store",
choices=sslversion_choices,
help="Set supported SSL/TLS version for server connections. "
"SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure."
)
ssl_verify_upstream_trusted_cadir=options.ssl_verify_upstream_trusted_cadir,
ssl_verify_upstream_trusted_ca=options.ssl_verify_upstream_trusted_ca
)

View File

@ -5,6 +5,8 @@ import sys
import socket
from libmproxy.protocol2.layer import Kill
from netlib import tcp
from netlib.http.http1 import HTTP1Protocol
from netlib.tcp import NetLibError
from ..protocol.handle import protocol_handler
from .. import protocol2
@ -82,11 +84,31 @@ class ConnectionHandler2:
self.channel
)
# FIXME: properly parse config
if self.config.mode == "upstream":
root_layer = protocol2.HttpUpstreamProxy(root_context, ("localhost", 8081))
else:
mode = self.config.mode
if mode == "upstream":
root_layer = protocol2.HttpUpstreamProxy(
root_context,
self.config.upstream_server.address
)
elif mode == "transparent":
root_layer = protocol2.TransparentProxy(root_context)
elif mode == "reverse":
client_tls = self.config.upstream_server.scheme.startswith("https")
server_tls = self.config.upstream_server.scheme.endswith("https")
root_layer = protocol2.ReverseProxy(
root_context,
self.config.upstream_server.address,
client_tls,
server_tls
)
elif mode == "socks5":
root_layer = protocol2.Socks5Proxy(root_context)
elif mode == "regular":
root_layer = protocol2.HttpProxy(root_context)
elif callable(mode): # pragma: nocover
root_layer = mode(root_context)
else: # pragma: nocover
raise ValueError("Unknown proxy mode: %s" % mode)
try:
root_layer()
@ -94,6 +116,14 @@ class ConnectionHandler2:
self.log("Connection killed", "info")
except ProtocolException as e:
self.log(e, "info")
# If an error propagates to the topmost level,
# we send an HTTP error response, which is both
# understandable by HTTP clients and humans.
try:
error_response = protocol2.make_error_response(502, repr(e))
self.client_conn.send(HTTP1Protocol().assemble(error_response))
except NetLibError:
pass
except Exception:
self.log(traceback.format_exc(), "error")
print(traceback.format_exc(), file=sys.stderr)

View File

@ -38,15 +38,15 @@ def test_parse_replace_hook():
def test_parse_server_spec():
tutils.raises("Invalid server specification", cmdline.parse_server_spec, "")
assert cmdline.parse_server_spec(
"http://foo.com:88") == [False, False, "foo.com", 88]
"http://foo.com:88") == ("http", ("foo.com", 88))
assert cmdline.parse_server_spec(
"http://foo.com") == [False, False, "foo.com", 80]
"http://foo.com") == ("http", ("foo.com", 80))
assert cmdline.parse_server_spec(
"https://foo.com") == [True, True, "foo.com", 443]
"https://foo.com") == ("https", ("foo.com", 443))
assert cmdline.parse_server_spec_special(
"https2http://foo.com") == [True, False, "foo.com", 80]
"https2http://foo.com") == ("https2http", ("foo.com", 80))
assert cmdline.parse_server_spec_special(
"http2https://foo.com") == [False, True, "foo.com", 443]
"http2https://foo.com") == ("http2https", ("foo.com", 443))
tutils.raises(
"Invalid server specification",
cmdline.parse_server_spec,
@ -55,6 +55,10 @@ def test_parse_server_spec():
"Invalid server specification",
cmdline.parse_server_spec,
"http://")
tutils.raises(
"Invalid server specification",
cmdline.parse_server_spec,
"https2http://foo.com")
def test_parse_setheaders():

View File

@ -4,6 +4,7 @@ import os.path
from cStringIO import StringIO
import email.utils
import mock
from libmproxy.cmdline import parse_server_spec
import netlib.utils
from netlib import odict
@ -672,11 +673,8 @@ class TestSerialize:
s = flow.State()
conf = ProxyConfig(
mode="reverse",
upstream_server=[
True,
True,
"use-this-domain",
80])
upstream_server=("https", ("use-this-domain", 80))
)
fm = flow.FlowMaster(DummyServer(conf), s)
fm.load_flows(r)
assert s.flows[0].request.host == "use-this-domain"

View File

@ -97,13 +97,7 @@ class TestProcessProxyOptions:
self.assert_err("expected one argument", "-U")
self.assert_err("Invalid server specification", "-U", "upstream")
self.assert_noerr("--spoof")
self.assert_noerr("--ssl-spoof")
self.assert_noerr("--spoofed-port", "443")
self.assert_err("expected one argument", "--spoofed-port")
self.assert_err("mutually exclusive", "-R", "http://localhost", "-T")
self.assert_err("not allowed with", "-R", "http://localhost", "-T")
def test_client_certs(self):
with tutils.tmpdir() as cadir:

View File

@ -464,60 +464,11 @@ class TestSocks5(tservers.SocksModeTest):
assert "SOCKS5 mode failure" in f.content
class TestSpoof(tservers.SpoofModeTest):
def test_http(self):
alist = (
("localhost", self.server.port),
("127.0.0.1", self.server.port)
)
for a in alist:
self.server.clear_log()
p = self.pathoc()
f = p.request("get:/p/304:h'Host'='%s:%s'" % a)
assert self.server.last_log()
assert f.status_code == 304
l = self.master.state.view[-1]
assert l.server_conn.address
assert l.server_conn.address.host == a[0]
assert l.server_conn.address.port == a[1]
def test_http_without_host(self):
p = self.pathoc()
f = p.request("get:/p/304:r")
assert f.status_code == 400
class TestSSLSpoof(tservers.SSLSpoofModeTest):
def test_https(self):
alist = (
("localhost", self.server.port),
("127.0.0.1", self.server.port)
)
for a in alist:
self.server.clear_log()
self.config.mode.sslport = a[1]
p = self.pathoc(sni=a[0])
f = p.request("get:/p/304")
assert self.server.last_log()
assert f.status_code == 304
l = self.master.state.view[-1]
assert l.server_conn.address
assert l.server_conn.address.host == a[0]
assert l.server_conn.address.port == a[1]
def test_https_without_sni(self):
a = ("localhost", self.server.port)
self.config.mode.sslport = a[1]
p = self.pathoc(sni=None)
f = p.request("get:/p/304")
assert f.status_code == 400
class TestHttps2Http(tservers.ReverseProxTest):
@classmethod
def get_proxy_config(cls):
d = super(TestHttps2Http, cls).get_proxy_config()
d["upstream_server"][0] = True
d["upstream_server"] = ("https2http", d["upstream_server"][1])
return d
def pathoc(self, ssl, sni=None):
@ -541,7 +492,7 @@ class TestHttps2Http(tservers.ReverseProxTest):
def test_http(self):
p = self.pathoc(ssl=False)
assert p.request("get:'/p/200'").status_code == 400
assert p.request("get:'/p/200'").status_code == 502
class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin):

View File

@ -1,6 +1,5 @@
import os.path
import threading
import Queue
import shutil
import tempfile
import flask
@ -130,7 +129,6 @@ class ProxTestBase(object):
no_upstream_cert = cls.no_upstream_cert,
cadir = cls.cadir,
authenticator = cls.authenticator,
ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []),
clientcerts = tutils.test_data.path("data/clientcert") if cls.clientcerts else None
)
@ -235,12 +233,10 @@ class ReverseProxTest(ProxTestBase):
@classmethod
def get_proxy_config(cls):
d = ProxTestBase.get_proxy_config()
d["upstream_server"] = [
True if cls.ssl else False,
True if cls.ssl else False,
"127.0.0.1",
cls.server.port
]
d["upstream_server"] = (
"https" if cls.ssl else "http",
("127.0.0.1", cls.server.port)
)
d["mode"] = "reverse"
return d
@ -360,7 +356,7 @@ class ChainProxTest(ProxTestBase):
if cls.chain: # First proxy is in normal mode.
d.update(
mode="upstream",
upstream_server=(False, False, "127.0.0.1", cls.chain[0].port)
upstream_server=("http", ("127.0.0.1", cls.chain[0].port))
)
return d