add generic tcp proxying, fix #374

This commit is contained in:
Maximilian Hils 2014-10-18 18:29:35 +02:00
parent 52b29d4926
commit e114858438
18 changed files with 250 additions and 88 deletions

View File

@ -17,6 +17,7 @@
$!nav("serverreplay.html", this, state)!$
$!nav("setheaders.html", this, state)!$
$!nav("passthrough.html", this, state)!$
$!nav("tcpproxy.html", this, state)!$
$!nav("sticky.html", this, state)!$
$!nav("reverseproxy.html", this, state)!$
$!nav("upstreamproxy.html", this, state)!$

View File

@ -12,6 +12,7 @@ pages = [
Page("setheaders.html", "Set Headers"),
Page("serverreplay.html", "Server-side replay"),
Page("sticky.html", "Sticky cookies and auth"),
Page("tcpproxy.html", "TCP Proxy"),
Page("upstreamcerts.html", "Upstream Certs"),
Page("upstreamproxy.html", "Upstream proxy mode"),
]

View File

@ -1,13 +1,12 @@
There are a couple of reasons why you may want to exempt some traffic from mitmproxy's interception mechanism:
There are two main reasons why you may want to exempt some traffic from mitmproxy's interception mechanism:
- **Certificate pinning:** Some traffic is is protected using
[certificate pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's
interception leads to errors. For example, Windows Update or the Apple App Store fail to work if mitmproxy is active.
- **Non-HTTP traffic:** WebSockets or other non-http protocols are not supported by mitmproxy yet. You can exempt the
domain from processing, which would otherwise fail.
- **Convenience:** You really don't care about some parts of the traffic and just want them to go away.
If you want to ignore traffic from mitmproxy's processing because of large response bodies, check out the
If you want to peek into (SSL-protected) non-HTTP connections, check out the [tcp proxy](@!urlTo("tcpproxy.html")!@) feature.
If you want to ignore traffic from mitmproxy's processing because of large response bodies, take a look at the
[response streaming](@!urlTo("responsestreaming.html")!@) feature.
## How it works
@ -74,4 +73,9 @@ Here are some other examples for ignore patterns:
--ignore 17\.178\.\d+\.\d+:443
</pre>
### See Also
- [TCP Proxy](@!urlTo("tcpproxy.html")!@)
- [Response Streaming](@!urlTo("responsestreaming.html")!@)
[^explicithttp]: This stems from an limitation of explicit HTTP proxying: A single connection can be re-used for multiple target domains - a <code>GET http://example.com/</code> request may be followed by a <code>GET http://evil.com/</code> request on the same connection. If we start to ignore the connection after the first request, we would miss the relevant second one.

View File

@ -47,4 +47,8 @@ When response streaming is enabled, portions of the code which would have otherw
on the response body will see an empty response body instead (<code>libmproxy.protocol.http.CONTENT_MISSING</code>). Any modifications will be ignored.
Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a <code>Transfer-Encoding:
chunked</code> header, the response will be streamed one chunk at a time.
chunked</code> header, the response will be streamed one chunk at a time.
### See Also
- [Ignore Domains](@!urlTo("passthrough.html")!@)

View File

@ -0,0 +1,30 @@
WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt hostnames from
processing, so that mitmproxy acts as a generic TCP forwarder. This feature is closely related to the
[ignore domains](@!urlTo("passthrough.html")!@) functionality, but differs in two important aspects:
- The raw TCP messages are printed to the event log.
- SSL connections will be intercepted.
Please note that message interception or modification are not possible yet.
If you are not interested in the raw TCP messages, you should use the ignore domains feature.
## How it works
<table class="table">
<tbody>
<tr>
<th width="20%">command-line</th> <td>--tcp HOST</td>
</tr>
<tr>
<th>mitmproxy shortcut</th> <td><b>T</b></td>
</tr>
</tbody>
</table>
For a detailed description on the structure of the hostname pattern, please refer to the [Ignore Domains](@!urlTo("passthrough.html")!@) feature.
### See Also
- [Ignore Domains](@!urlTo("passthrough.html")!@)
- [Response Streaming](@!urlTo("responsestreaming.html")!@)

View File

@ -263,7 +263,7 @@ def common_options(parser):
)
group.add_argument(
"-I", "--ignore",
action="append", type=str, dest="ignore", default=[],
action="append", type=str, dest="ignore_hosts", default=[],
metavar="HOST",
help="Ignore host and forward all traffic without processing it. "
"In transparent mode, it is recommended to use an IP address (range), not the hostname. "
@ -271,6 +271,13 @@ def common_options(parser):
"The supplied value is interpreted as a regular expression and matched on the ip or the hostname. "
"Can be passed multiple times. "
)
group.add_argument(
"--tcp",
action="append", type=str, dest="tcp_hosts", default=[],
metavar="HOST",
help="Generic TCP SSL proxy mode for all hosts that match the pattern. Similar to --ignore,"
"but SSL connections are intercepted. The communication contents are printed to the event log in verbose mode."
)
group.add_argument(
"-n",
action="store_true", dest="no_server",

View File

@ -129,10 +129,14 @@ class StatusBar(common.WWrap):
r.append(":%s in file]"%self.master.server_playback.count())
else:
r.append(":%s to go]"%self.master.server_playback.count())
if self.master.get_ignore():
if self.master.get_ignore_filter():
r.append("[")
r.append(("heading_key", "I"))
r.append("gnore:%d]"%len(self.master.get_ignore()))
r.append("gnore:%d]" % len(self.master.get_ignore_filter()))
if self.master.get_tcp_filter():
r.append("[")
r.append(("heading_key", "T"))
r.append("CP:%d]" % len(self.master.get_tcp_filter()))
if self.master.state.intercept_txt:
r.append("[")
r.append(("heading_key", "i"))
@ -798,9 +802,13 @@ class ConsoleMaster(flow.FlowMaster):
for command in commands:
self.load_script(command)
def edit_ignore(self, ignore):
def edit_ignore_filter(self, ignore):
patterns = (x[0] for x in ignore)
self.set_ignore(patterns)
self.set_ignore_filter(patterns)
def edit_tcp_filter(self, tcp):
patterns = (x[0] for x in tcp)
self.set_tcp_filter(patterns)
def loop(self):
changed = True
@ -860,10 +868,18 @@ class ConsoleMaster(flow.FlowMaster):
)
elif k == "I":
self.view_grideditor(
grideditor.IgnoreEditor(
grideditor.HostPatternEditor(
self,
[[x] for x in self.get_ignore()],
self.edit_ignore
[[x] for x in self.get_ignore_filter()],
self.edit_ignore_filter
)
)
elif k == "T":
self.view_grideditor(
grideditor.HostPatternEditor(
self,
[[x] for x in self.get_tcp_filter()],
self.edit_tcp_filter
)
)
elif k == "i":

View File

@ -495,8 +495,8 @@ class ScriptEditor(GridEditor):
return str(v)
class IgnoreEditor(GridEditor):
title = "Editing ignore patterns"
class HostPatternEditor(GridEditor):
title = "Editing host patterns"
columns = 1
headings = ("Regex (matched on hostname:port / ip:port)",)

View File

@ -119,6 +119,7 @@ class HelpView(urwid.ListBox):
("s", "add/remove scripts"),
("S", "server replay"),
("t", "set sticky cookie expression"),
("T", "set tcp proxying pattern"),
("u", "set sticky auth expression"),
]
text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))

View File

@ -11,7 +11,7 @@ import netlib.http
from . import controller, protocol, tnetstring, filt, script, version
from .onboarding import app
from .protocol import http, handle
from .proxy.config import parse_host_pattern
from .proxy.config import HostMatcher
import urlparse
ODict = odict.ODict
@ -515,11 +515,17 @@ class FlowMaster(controller.Master):
for script in self.scripts:
self.run_single_script_hook(script, name, *args, **kwargs)
def get_ignore(self):
return [i.pattern for i in self.server.config.ignore]
def get_ignore_filter(self):
return self.server.config.check_ignore.patterns
def set_ignore(self, ignore):
self.server.config.ignore = parse_host_pattern(ignore)
def set_ignore_filter(self, host_patterns):
self.server.config.check_ignore = HostMatcher(host_patterns)
def get_tcp_filter(self):
return self.server.config.check_tcp.patterns
def set_tcp_filter(self, host_patterns):
self.server.config.check_tcp = HostMatcher(host_patterns)
def set_stickycookie(self, txt):
if txt:
@ -787,7 +793,7 @@ class FlowReader:
v = ".".join(str(i) for i in data["version"])
raise FlowReadError("Incompatible serialized data version: %s"%v)
off = self.fo.tell()
yield handle.protocols[data["conntype"]]["flow"].from_state(data)
yield handle.protocols[data["type"]]["flow"].from_state(data)
except ValueError, v:
# Error is due to EOF
if self.fo.tell() == off and self.fo.read() == '':

View File

@ -1260,9 +1260,9 @@ class HTTPHandler(ProtocolHandler):
Returns False, if the connection should be closed immediately.
"""
address = tcp.Address.wrap(address)
if self.c.check_ignore_address(address):
if self.c.config.check_ignore(address):
self.c.log("Ignore host: %s:%s" % address(), "info")
TCPHandler(self.c).handle_messages()
TCPHandler(self.c, log=False).handle_messages()
return False
else:
self.expected_form_in = "relative"
@ -1274,6 +1274,11 @@ class HTTPHandler(ProtocolHandler):
self.c.establish_ssl(server=True, client=True)
self.c.log("Upgrade to SSL completed.", "debug")
if self.c.config.check_tcp(address):
self.c.log("Generic TCP mode for host: %s:%s" % address(), "info")
TCPHandler(self.c).handle_messages()
return False
return True
def authenticate(self, request):

View File

@ -59,8 +59,8 @@ class Flow(stateobject.StateObject):
A Flow is a collection of objects representing a single transaction.
This class is usually subclassed for each protocol, e.g. HTTPFlow.
"""
def __init__(self, conntype, client_conn, server_conn, live=None):
self.conntype = conntype
def __init__(self, type, client_conn, server_conn, live=None):
self.type = type
self.id = str(uuid.uuid4())
self.client_conn = client_conn
"""@type: ClientConnection"""
@ -78,7 +78,7 @@ class Flow(stateobject.StateObject):
error=Error,
client_conn=ClientConnection,
server_conn=ServerConnection,
conntype=str
type=str
)
def get_state(self, short=False):

View File

@ -13,6 +13,10 @@ class TCPHandler(ProtocolHandler):
chunk_size = 4096
def __init__(self, c, log=True):
super(TCPHandler, self).__init__(c)
self.log = log
def handle_messages(self):
self.c.establish_server_connection()
@ -63,26 +67,25 @@ class TCPHandler(ProtocolHandler):
# if one of the peers is over SSL, we need to send
# bytes/strings
if not src.ssl_established:
# only ssl to dst, i.e. we revc'd into buf but need
# bytes/string now.
# we revc'd into buf but need bytes/string now.
contents = buf[:size].tobytes()
self.c.log(
"%s %s\r\n%s" % (
direction, dst_str, cleanBin(contents)
),
"debug"
)
if self.log:
self.c.log(
"%s %s\r\n%s" % (
direction, dst_str, cleanBin(contents)
),
"info"
)
dst.connection.send(contents)
else:
# socket.socket.send supports raw bytearrays/memoryviews
self.c.log(
"%s %s\r\n%s" % (
direction,
dst_str,
cleanBin(buf.tobytes())
),
"debug"
)
if self.log:
self.c.log(
"%s %s\r\n%s" % (
direction, dst_str, cleanBin(buf.tobytes())
),
"info"
)
dst.connection.send(buf[:size])
except socket.error as e:
self.c.log("TCP connection closed unexpectedly.", "debug")

View File

@ -1,7 +1,7 @@
from __future__ import absolute_import
import os
import re
from netlib import http_auth, certutils
from netlib import http_auth, certutils, tcp
from .. import utils, platform, version
from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode
@ -10,8 +10,21 @@ CONF_BASENAME = "mitmproxy"
CONF_DIR = "~/.mitmproxy"
def parse_host_pattern(patterns):
return [re.compile(p, re.IGNORECASE) for p in patterns]
class HostMatcher(object):
def __init__(self, patterns=[]):
self.patterns = list(patterns)
self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns]
def __call__(self, address):
address = tcp.Address.wrap(address)
host = "%s:%s" % (address.host, address.port)
if any(rex.search(host) for rex in self.regexes):
return True
else:
return False
def __nonzero__(self):
return bool(self.patterns)
class ProxyConfig:
@ -19,7 +32,7 @@ class ProxyConfig:
confdir=CONF_DIR, clientcerts=None,
no_upstream_cert=False, body_size_limit=None,
mode=None, upstream_server=None, http_form_in=None, http_form_out=None,
authenticator=None, ignore=[],
authenticator=None, ignore_hosts=[], tcp_hosts=[],
ciphers=None, certs=[], certforward=False, ssl_ports=TRANSPARENT_SSL_PORTS):
self.host = host
self.port = port
@ -44,7 +57,8 @@ class ProxyConfig:
self.mode.http_form_in = http_form_in or self.mode.http_form_in
self.mode.http_form_out = http_form_out or self.mode.http_form_out
self.ignore = parse_host_pattern(ignore)
self.check_ignore = HostMatcher(ignore_hosts)
self.check_tcp = HostMatcher(tcp_hosts)
self.authenticator = authenticator
self.confdir = os.path.expanduser(confdir)
self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME)
@ -124,7 +138,8 @@ def process_proxy_options(parser, options):
upstream_server=upstream_server,
http_form_in=options.http_form_in,
http_form_out=options.http_form_out,
ignore=options.ignore,
ignore_hosts=options.ignore_hosts,
tcp_hosts=options.tcp_hosts,
authenticator=authenticator,
ciphers=options.ciphers,
certs=certs,

View File

@ -70,13 +70,15 @@ class ConnectionHandler:
# Can we already identify the target server and connect to it?
client_ssl, server_ssl = False, False
conn_kwargs = dict()
upstream_info = self.config.mode.get_upstream_server(self.client_conn)
if upstream_info:
self.set_server_address(upstream_info[2:])
client_ssl, server_ssl = upstream_info[:2]
if self.check_ignore_address(self.server_conn.address):
if self.config.check_ignore(self.server_conn.address):
self.log("Ignore host: %s:%s" % self.server_conn.address(), "info")
self.conntype = "tcp"
conn_kwargs["log"] = False
client_ssl, server_ssl = False, False
else:
pass # No upstream info from the metadata: upstream info in the protocol (e.g. HTTP absolute-form)
@ -90,15 +92,19 @@ class ConnectionHandler:
if client_ssl or server_ssl:
self.establish_ssl(client=client_ssl, server=server_ssl)
if self.config.check_tcp(self.server_conn.address):
self.log("Generic TCP mode for host: %s:%s" % self.server_conn.address(), "info")
self.conntype = "tcp"
# Delegate handling to the protocol handler
protocol_handler(self.conntype)(self).handle_messages()
protocol_handler(self.conntype)(self, **conn_kwargs).handle_messages()
self.del_server_connection()
self.log("clientdisconnect", "info")
self.channel.tell("clientdisconnect", self)
except ProxyError as e:
protocol_handler(self.conntype)(self).handle_error(e)
protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e)
except Exception:
import traceback, sys
@ -119,14 +125,6 @@ class ConnectionHandler:
self.server_conn = None
self.sni = None
def check_ignore_address(self, address):
address = tcp.Address.wrap(address)
host = "%s:%s" % (address.host, address.port)
if host and any(rex.search(host) for rex in self.config.ignore):
return True
else:
return False
def set_server_address(self, address):
"""
Sets a new server address with the given priority.

View File

@ -93,7 +93,7 @@
"clientcert": null,
"ssl_established": true
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -259,7 +259,7 @@
"clientcert": null,
"ssl_established": true
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -425,7 +425,7 @@
"clientcert": null,
"ssl_established": true
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -595,7 +595,7 @@
"clientcert": null,
"ssl_established": true
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -765,7 +765,7 @@
"clientcert": null,
"ssl_established": true
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -919,7 +919,7 @@
"clientcert": null,
"ssl_established": false
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -1057,7 +1057,7 @@
"clientcert": null,
"ssl_established": false
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -1195,7 +1195,7 @@
"clientcert": null,
"ssl_established": false
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -1329,7 +1329,7 @@
"clientcert": null,
"ssl_established": false
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -1483,7 +1483,7 @@
"clientcert": null,
"ssl_established": false
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -1633,7 +1633,7 @@
"clientcert": null,
"ssl_established": false
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -1767,7 +1767,7 @@
"clientcert": null,
"ssl_established": false
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -1901,7 +1901,7 @@
"clientcert": null,
"ssl_established": false
},
"conntype": "http",
"type": "http",
"version": [
0,
11
@ -2027,7 +2027,7 @@
"clientcert": null,
"ssl_established": false
},
"conntype": "http",
"type": "http",
"version": [
0,
11

View File

@ -5,8 +5,8 @@ import mock
from libmproxy import filt, protocol, controller, utils, tnetstring, flow
from libmproxy.protocol.primitives import Error, Flow
from libmproxy.protocol.http import decoded, CONTENT_MISSING
from libmproxy.proxy.connection import ClientConnection, ServerConnection
from netlib import tcp
from libmproxy.proxy.connection import ClientConnection
from libmproxy.proxy.config import HostMatcher
import tutils
@ -584,11 +584,11 @@ class TestFlowMaster:
def test_getset_ignore(self):
p = mock.Mock()
p.config.ignore = []
p.config.check_ignore = HostMatcher()
fm = flow.FlowMaster(p, flow.State())
assert not fm.get_ignore()
fm.set_ignore(["^apple\.com:", ":443$"])
assert fm.get_ignore()
assert not fm.get_ignore_filter()
fm.set_ignore_filter(["^apple\.com:", ":443$"])
assert fm.get_ignore_filter()
def test_replay(self):
s = flow.State()

View File

@ -1,5 +1,5 @@
import socket, time
from libmproxy.proxy.config import parse_host_pattern
from libmproxy.proxy.config import HostMatcher
from netlib import tcp, http_auth, http
from libpathod import pathoc, pathod
from netlib.certutils import SSLCert
@ -79,11 +79,14 @@ class CommonMixin:
class TcpMixin:
def _ignore_on(self):
ignore = parse_host_pattern([".+:%s" % self.server.port])[0]
self.config.ignore.append(ignore)
assert not hasattr(self, "_ignore_backup")
self._ignore_backup = self.config.check_ignore
self.config.check_ignore = HostMatcher([".+:%s" % self.server.port] + self.config.check_ignore.patterns)
def _ignore_off(self):
self.config.ignore.pop()
assert hasattr(self, "_ignore_backup")
self.config.check_ignore = self._ignore_backup
del self._ignore_backup
def test_ignore(self):
spec = '304:h"Alternate-Protocol"="mitmproxy-will-remove-this"'
@ -114,6 +117,40 @@ class TcpMixin:
tutils.raises("invalid server response", self.pathod, spec) # pathoc tries to parse answer as HTTP
self._ignore_off()
def _tcpproxy_on(self):
assert not hasattr(self, "_tcpproxy_backup")
self._tcpproxy_backup = self.config.check_tcp
self.config.check_tcp = HostMatcher([".+:%s" % self.server.port] + self.config.check_tcp.patterns)
def _tcpproxy_off(self):
assert hasattr(self, "_tcpproxy_backup")
self.config.check_ignore = self._tcpproxy_backup
del self._tcpproxy_backup
def test_tcp(self):
spec = '304:h"Alternate-Protocol"="mitmproxy-will-remove-this"'
n = self.pathod(spec)
self._tcpproxy_on()
i = self.pathod(spec)
i2 = self.pathod(spec)
self._tcpproxy_off()
assert i.status_code == i2.status_code == n.status_code == 304
assert "Alternate-Protocol" in i.headers
assert "Alternate-Protocol" in i2.headers
assert "Alternate-Protocol" not in n.headers
# Test that we get the original SSL cert
if self.ssl:
i_cert = SSLCert(i.sslinfo.certchain[0])
i2_cert = SSLCert(i2.sslinfo.certchain[0])
n_cert = SSLCert(n.sslinfo.certchain[0])
assert i_cert == i2_cert == n_cert
# Make sure that TCP messages are in the event log.
assert any("mitmproxy-will-remove-this" in m for m in self.master.log)
class AppMixin:
def test_app(self):
@ -579,16 +616,50 @@ class TestUpstreamProxy(tservers.HTTPUpstreamProxTest, CommonMixin, AppMixin):
class TestUpstreamProxySSL(tservers.HTTPUpstreamProxTest, CommonMixin, TcpMixin):
ssl = True
def _host_pattern_on(self, attr):
"""
Updates config.check_tcp or check_ignore, depending on attr.
"""
assert not hasattr(self, "_ignore_%s_backup" % attr)
backup = []
for proxy in self.chain:
old_matcher = getattr(proxy.tmaster.server.config, "check_%s" % attr)
backup.append(old_matcher)
setattr(
proxy.tmaster.server.config,
"check_%s" % attr,
HostMatcher([".+:%s" % self.server.port] + old_matcher.patterns)
)
setattr(self, "_ignore_%s_backup" % attr, backup)
def _host_pattern_off(self, attr):
backup = getattr(self, "_ignore_%s_backup" % attr)
for proxy in reversed(self.chain):
setattr(
proxy.tmaster.server.config,
"check_%s" % attr,
backup.pop()
)
assert not backup
delattr(self, "_ignore_%s_backup" % attr)
def _ignore_on(self):
super(TestUpstreamProxySSL, self)._ignore_on()
ignore = parse_host_pattern([".+:%s" % self.server.port])[0]
for proxy in self.chain:
proxy.tmaster.server.config.ignore.append(ignore)
self._host_pattern_on("ignore")
def _ignore_off(self):
super(TestUpstreamProxySSL, self)._ignore_off()
for proxy in self.chain:
proxy.tmaster.server.config.ignore.pop()
self._host_pattern_off("ignore")
def _tcpproxy_on(self):
super(TestUpstreamProxySSL, self)._tcpproxy_on()
self._host_pattern_on("tcp")
def _tcpproxy_off(self):
super(TestUpstreamProxySSL, self)._tcpproxy_off()
self._host_pattern_off("tcp")
def test_simple(self):
p = self.pathoc()