Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-23 00:01:36 +00:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
Maximilian Hils 2013-12-08 14:14:31 +01:00
commit e3c69fd105
16 changed files with 162 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc-src/scripting/inlinescripts.html
View File

@ -25,7 +25,7 @@ The new header will be added to all responses passing through the proxy.
## Events
### start(ScriptContext)
### start(ScriptContext, argv)
Called once on startup, before any other events.

12
doc-src/transparent/osx.html
View File

@ -67,3 +67,15 @@ rdr on en2 inet proto tcp to any port 443 -> 127.0.0.1 port 8080
</ol>
Note that the **rdr** rules in the pf.conf given above only apply to inbound
traffic. This means that they will NOT redirect traffic coming from the box
running pf itself. We can't distinguish between an outbound connection from a
non-mitmproxy app, and an outbound connection from mitmproxy itself - if you
want to intercept your OSX traffic, you should use an external host to run
mitmproxy. None the less, pf is flexible to cater for a range of creative
possibilities, like intercepting traffic emanating from VMs. See the
**pf.conf** man page for more.

3
examples/stub.py
View File

@ -1,8 +1,7 @@
"""
This is a script stub, with definitions for all events.
"""
def start(ctx):
def start(ctx, argv):
"""
Called once on script startup, before any other events.
"""

14
libmproxy/cmdline.py
View File

@ -1,6 +1,8 @@
import proxy
import re, filt
import argparse
import shlex
import os
APP_DOMAIN = "mitm"
APP_IP = "1.1.1.1"
@ -143,7 +145,7 @@ def get_common_options(options):
replacements = reps,
setheaders = setheaders,
server_replay = options.server_replay,
script = options.script,
scripts = options.scripts,
stickycookie = stickycookie,
stickyauth = stickyauth,
showhost = options.showhost,
@ -189,6 +191,11 @@ def common_options(parser):
action="store", dest="reverse_proxy", default=None,
help="Reverse proxy to upstream server: http[s]://host[:port]"
)
parser.add_argument(
"-F",
action="store", dest="forward_proxy", default=None,
help="Proxy to unconditionally forward to: http[s]://host[:port]"
)
parser.add_argument(
"-q",
action="store_true", dest="quiet",
@ -201,8 +208,9 @@ def common_options(parser):
)
parser.add_argument(
"-s",
action="store", dest="script", default=None,
help="Run a script."
action="append", type=lambda x: shlex.split(x,posix=(os.name != "nt")), dest="scripts", default=[],
metavar='"script.py --bar"',
help="Run a script. Surround with quotes to pass script arguments. Can be passed multiple times."
)
parser.add_argument(
"-t",

2
libmproxy/console/__init__.py
View File

@ -459,7 +459,7 @@ class ConsoleMaster(flow.FlowMaster):
self._run_script_method("response", s, f)
if f.error:
self._run_script_method("error", s, f)
s.run("done")
s.unload()
self.refresh_flow(f)
self.state.last_script = path

2
libmproxy/console/common.py
View File

@ -190,7 +190,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
delta = f.response.timestamp_end - f.response.timestamp_start
size = len(f.response.content) + f.response.get_header_size()
rate = utils.pretty_size(size / delta)
rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) )
d.update(dict(
resp_code = f.response.code,

10
libmproxy/dump.py
View File

@ -24,7 +24,7 @@ class Options(object):
"rheaders",
"setheaders",
"server_replay",
"script",
"scripts",
"showhost",
"stickycookie",
"stickyauth",
@ -109,8 +109,8 @@ class DumpMaster(flow.FlowMaster):
not options.keepserving
)
if options.script:
err = self.load_script(options.script)
for script_argv in options.scripts:
err = self.load_script(script_argv)
if err:
raise DumpError(err)
@ -221,8 +221,8 @@ class DumpMaster(flow.FlowMaster):
def run(self): # pragma: no cover
if self.o.rfile and not self.o.keepserving:
if self.script:
self.load_script(None)
for script in self.scripts:
self.unload_script(script)
return
try:
return flow.FlowMaster.run(self)

60
libmproxy/flow.py
View File

@ -1349,7 +1349,7 @@ class FlowMaster(controller.Master):
self.server_playback = None
self.client_playback = None
self.kill_nonreplay = False
self.script = None
self.scripts = []
self.pause_scripts = False
self.stickycookie_state = False
@ -1385,37 +1385,43 @@ class FlowMaster(controller.Master):
"""
pass
def get_script(self, path):
def get_script(self, script_argv):
"""
Returns an (error, script) tuple.
"""
s = script.Script(path, ScriptContext(self))
s = script.Script(script_argv, ScriptContext(self))
try:
s.load()
except script.ScriptError, v:
return (v.args[0], None)
ret = s.run("start")
if not ret[0] and ret[1]:
return ("Error in script start:\n\n" + ret[1][1], None)
return (None, s)
def load_script(self, path):
def unload_script(self,script):
script.unload()
self.scripts.remove(script)
def load_script(self, script_argv):
"""
Loads a script. Returns an error description if something went
wrong. If path is None, the current script is terminated.
wrong.
"""
if path is None:
self.run_script_hook("done")
self.script = None
r = self.get_script(script_argv)
if r[0]:
return r[0]
else:
r = self.get_script(path)
if r[0]:
return r[0]
else:
if self.script:
self.run_script_hook("done")
self.script = r[1]
self.scripts.append(r[1])
def run_single_script_hook(self, script, name, *args, **kwargs):
if script and not self.pause_scripts:
ret = script.run(name, *args, **kwargs)
if not ret[0] and ret[1]:
e = "Script error:\n" + ret[1][1]
self.add_event(e, "error")
def run_script_hook(self, name, *args, **kwargs):
for script in self.scripts:
self.run_single_script_hook(script, name, *args, **kwargs)
def set_stickycookie(self, txt):
if txt:
flt = filt.parse(txt)
@ -1565,13 +1571,6 @@ class FlowMaster(controller.Master):
if block:
rt.join()
def run_script_hook(self, name, *args, **kwargs):
if self.script and not self.pause_scripts:
ret = self.script.run(name, *args, **kwargs)
if not ret[0] and ret[1]:
e = "Script error:\n" + ret[1][1]
self.add_event(e, "error")
def handle_clientconnect(self, cc):
self.run_script_hook("clientconnect", cc)
cc.reply()
@ -1580,6 +1579,13 @@ class FlowMaster(controller.Master):
self.run_script_hook("clientdisconnect", r)
r.reply()
def handle_serverconnection(self, sc):
# To unify the mitmproxy script API, we call the script hook "serverconnect" rather than "serverconnection".
# As things are handled differently in libmproxy (ClientConnect + ClientDisconnect vs ServerConnection class),
# there is no "serverdisonnect" event at the moment.
self.run_script_hook("serverconnect", sc)
sc.reply()
def handle_error(self, r):
f = self.state.add_error(r)
if f:
@ -1613,8 +1619,8 @@ class FlowMaster(controller.Master):
return f
def shutdown(self):
if self.script:
self.load_script(None)
for script in self.scripts:
self.unload_script(script)
controller.Master.shutdown(self)
if self.stream:
for i in self.state._flow_list:

60
libmproxy/proxy.py
View File

@ -23,13 +23,14 @@ class Log:
class ProxyConfig:
def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, transparent_proxy=None, authenticator=None):
def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, forward_proxy=None, transparent_proxy=None, authenticator=None):
self.certfile = certfile
self.cacert = cacert
self.clientcerts = clientcerts
self.no_upstream_cert = no_upstream_cert
self.body_size_limit = body_size_limit
self.reverse_proxy = reverse_proxy
self.forward_proxy = forward_proxy
self.transparent_proxy = transparent_proxy
self.authenticator = authenticator
self.certstore = certutils.CertStore()
@ -158,6 +159,7 @@ class ProxyHandler(tcp.BaseHandler):
if not self.server_conn:
try:
self.server_conn = ServerConnection(self.config, scheme, host, port, sni)
self.channel.ask(self.server_conn)
self.server_conn.connect()
except tcp.NetLibError, v:
raise ProxyError(502, v)
@ -219,7 +221,12 @@ class ProxyHandler(tcp.BaseHandler):
# the case, we want to reconnect without sending an error
# to the client.
while 1:
sc = self.get_server_connection(cc, scheme, host, port, self.sni)
if self.config.forward_proxy:
forward_scheme, forward_host, forward_port = self.config.forward_proxy
sc = self.get_server_connection(cc, forward_scheme, forward_host, forward_port, self.sni)
else:
sc = self.get_server_connection(cc, scheme, host, port, self.sni)
sc.send(request)
if sc.requestcount == 1: # add timestamps only for first request (others are not directly affected)
request.tcp_setup_timestamp = sc.tcp_setup_timestamp
@ -258,13 +265,13 @@ class ProxyHandler(tcp.BaseHandler):
else:
response = response_reply
self.send_response(response)
if request and http.request_connection_close(request.httpversion, request.headers):
if request and http.connection_close(request.httpversion, request.headers):
return
# We could keep the client connection when the server
# connection needs to go away. However, we want to mimic
# behaviour as closely as possible to the client, so we
# disconnect.
if http.response_connection_close(response.httpversion, response.headers):
if http.connection_close(response.httpversion, response.headers):
return
except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e:
if hasattr(e, "code"):
@ -310,6 +317,17 @@ class ProxyHandler(tcp.BaseHandler):
raise ProxyError(502, "Unable to generate dummy cert.")
return ret
def establish_ssl(self, client_conn, host, port):
dummycert = self.find_cert(client_conn, host, port, host)
sni = HandleSNI(
self, client_conn, host, port,
dummycert, self.config.certfile or self.config.cacert
)
try:
self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
def get_line(self, fp):
"""
Get a line, possibly preceded by a blank.
@ -329,15 +347,7 @@ class ProxyHandler(tcp.BaseHandler):
if port in self.config.transparent_proxy["sslports"]:
scheme = "https"
if not self.ssl_established:
dummycert = self.find_cert(client_conn, host, port, host)
sni = HandleSNI(
self, client_conn, host, port,
dummycert, self.config.certfile or self.config.cacert
)
try:
self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
self.establish_ssl(client_conn, host, port)
else:
scheme = "http"
line = self.get_line(self.rfile)
@ -372,15 +382,7 @@ class ProxyHandler(tcp.BaseHandler):
'\r\n'
)
self.wfile.flush()
dummycert = self.find_cert(client_conn, host, port, host)
sni = HandleSNI(
self, client_conn, host, port,
dummycert, self.config.certfile or self.config.cacert
)
try:
self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
self.establish_ssl(client_conn, host, port)
self.proxy_connect_state = (host, port, httpversion)
line = self.rfile.readline(line)
@ -414,10 +416,12 @@ class ProxyHandler(tcp.BaseHandler):
)
def read_request_reverse(self, client_conn):
scheme, host, port = self.config.reverse_proxy
if scheme.lower() == "https" and not self.ssl_established:
self.establish_ssl(client_conn, host, port)
line = self.get_line(self.rfile)
if line == "":
return None
scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
@ -427,7 +431,7 @@ class ProxyHandler(tcp.BaseHandler):
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn, httpversion, host, port, "http", method, path, headers, content,
client_conn, httpversion, host, port, scheme, method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
@ -594,6 +598,13 @@ def process_proxy_options(parser, options):
else:
rp = None
if options.forward_proxy:
fp = utils.parse_proxy_spec(options.forward_proxy)
if not fp:
return parser.error("Invalid forward proxy specification: %s"%options.forward_proxy)
else:
fp = None
if options.clientcerts:
options.clientcerts = os.path.expanduser(options.clientcerts)
if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts):
@ -623,6 +634,7 @@ def process_proxy_options(parser, options):
body_size_limit = body_size_limit,
no_upstream_cert = options.no_upstream_cert,
reverse_proxy = rp,
forward_proxy = fp,
transparent_proxy = trans,
authenticator = authenticator
)

20
libmproxy/script.py
View File

@ -8,12 +8,12 @@ class Script:
"""
The instantiator should do something along this vein:
s = Script(path, master)
s = Script(argv, master)
s.load()
s.run("start")
"""
def __init__(self, path, ctx):
self.path, self.ctx = path, ctx
def __init__(self, argv, ctx):
self.argv = argv
self.ctx = ctx
self.ns = None
def load(self):
@ -23,17 +23,21 @@ class Script:
Raises ScriptError on failure, with argument equal to an error
message that may be a formatted traceback.
"""
path = os.path.expanduser(self.path)
path = os.path.expanduser(self.argv[0])
if not os.path.exists(path):
raise ScriptError("No such file: %s"%self.path)
raise ScriptError("No such file: %s" % path)
if not os.path.isfile(path):
raise ScriptError("Not a file: %s"%self.path)
raise ScriptError("Not a file: %s" % path)
ns = {}
try:
execfile(path, ns, ns)
self.ns = ns
self.run("start", self.argv)
except Exception, v:
raise ScriptError(traceback.format_exc(v))
self.ns = ns
def unload(self):
return self.run("done")
def run(self, name, *args, **kwargs):
"""

8
test/scripts/a.py
View File

@ -1,5 +1,13 @@
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--var', type=int)
var = 0
def start(ctx, argv):
global var
var = parser.parse_args(argv[1:]).var
def here(ctx):
global var
var += 1

2
test/scripts/starterr.py
View File

@ -1,3 +1,3 @@
def start(ctx):
def start(ctx, argv):
raise ValueError

13
test/test_cmdline.py
View File

@ -1,6 +1,7 @@
import argparse
from libmproxy import cmdline
import tutils
import os.path
def test_parse_replace_hook():
@ -39,6 +40,18 @@ def test_parse_setheaders():
x = cmdline.parse_setheader("/foo/bar/voing")
assert x == ("foo", "bar", "voing")
def test_shlex():
"""
shlex.split assumes posix=True by default, we do manual detection for windows.
Test whether script paths are parsed correctly
"""
absfilepath = os.path.normcase(os.path.abspath(__file__))
parser = argparse.ArgumentParser()
cmdline.common_options(parser)
opts = parser.parse_args(args=["-s",absfilepath])
assert os.path.isfile(opts.scripts[0][0])
def test_common():
parser = argparse.ArgumentParser()

21
test/test_flow.py
View File

@ -544,9 +544,11 @@ class TestFlowMaster:
fm = flow.FlowMaster(None, s)
assert not fm.load_script(tutils.test_data.path("scripts/a.py"))
assert not fm.load_script(tutils.test_data.path("scripts/a.py"))
assert not fm.load_script(None)
assert not fm.unload_script(fm.scripts[0])
assert not fm.unload_script(fm.scripts[0])
assert fm.load_script("nonexistent")
assert "ValueError" in fm.load_script(tutils.test_data.path("scripts/starterr.py"))
assert len(fm.scripts) == 0
def test_replay(self):
s = flow.State()
@ -572,20 +574,27 @@ class TestFlowMaster:
assert not fm.load_script(tutils.test_data.path("scripts/all.py"))
req = tutils.treq()
fm.handle_clientconnect(req.client_conn)
assert fm.script.ns["log"][-1] == "clientconnect"
assert fm.scripts[0].ns["log"][-1] == "clientconnect"
f = fm.handle_request(req)
assert fm.script.ns["log"][-1] == "request"
assert fm.scripts[0].ns["log"][-1] == "request"
resp = tutils.tresp(req)
fm.handle_response(resp)
assert fm.script.ns["log"][-1] == "response"
assert fm.scripts[0].ns["log"][-1] == "response"
#load second script
assert not fm.load_script(tutils.test_data.path("scripts/all.py"))
assert len(fm.scripts) == 2
dc = flow.ClientDisconnect(req.client_conn)
dc.reply = controller.DummyReply()
fm.handle_clientdisconnect(dc)
assert fm.script.ns["log"][-1] == "clientdisconnect"
assert fm.scripts[0].ns["log"][-1] == "clientdisconnect"
assert fm.scripts[1].ns["log"][-1] == "clientdisconnect"
#unload first script
fm.unload_script(fm.scripts[0])
assert len(fm.scripts) == 1
err = flow.Error(f.request, "msg")
err.reply = controller.DummyReply()
fm.handle_error(err)
assert fm.script.ns["log"][-1] == "error"
assert fm.scripts[0].ns["log"][-1] == "error"
def test_duplicate_flow(self):
s = flow.State()

21
test/test_script.py
View File

@ -1,5 +1,7 @@
from libmproxy import script, flow
import tutils
import shlex
import os
class TestScript:
def test_simple(self):
@ -7,11 +9,12 @@ class TestScript:
fm = flow.FlowMaster(None, s)
ctx = flow.ScriptContext(fm)
p = script.Script(tutils.test_data.path("scripts/a.py"), ctx)
p = script.Script(shlex.split(tutils.test_data.path("scripts/a.py")+" --var 40", posix=(os.name != "nt")), ctx)
p.load()
assert "here" in p.ns
assert p.run("here") == (True, 1)
assert p.run("here") == (True, 2)
assert p.run("here") == (True, 41)
assert p.run("here") == (True, 42)
ret = p.run("errargs")
assert not ret[0]
@ -19,12 +22,12 @@ class TestScript:
# Check reload
p.load()
assert p.run("here") == (True, 1)
assert p.run("here") == (True, 41)
def test_duplicate_flow(self):
s = flow.State()
fm = flow.FlowMaster(None, s)
fm.load_script(tutils.test_data.path("scripts/duplicate_flow.py"))
fm.load_script([tutils.test_data.path("scripts/duplicate_flow.py")])
r = tutils.treq()
fm.handle_request(r)
assert fm.state.flow_count() == 2
@ -37,25 +40,25 @@ class TestScript:
ctx = flow.ScriptContext(fm)
s = script.Script("nonexistent", ctx)
s = script.Script(["nonexistent"], ctx)
tutils.raises(
"no such file",
s.load
)
s = script.Script(tutils.test_data.path("scripts"), ctx)
s = script.Script([tutils.test_data.path("scripts")], ctx)
tutils.raises(
"not a file",
s.load
)
s = script.Script(tutils.test_data.path("scripts/syntaxerr.py"), ctx)
s = script.Script([tutils.test_data.path("scripts/syntaxerr.py")], ctx)
tutils.raises(
script.ScriptError,
s.load
)
s = script.Script(tutils.test_data.path("scripts/loaderr.py"), ctx)
s = script.Script([tutils.test_data.path("scripts/loaderr.py")], ctx)
tutils.raises(
script.ScriptError,
s.load

10
test/test_server.py
View File

@ -191,16 +191,6 @@ class TestHTTPS(tservers.HTTPProxTest, CommonMixin):
assert p.request("get:/:i0,'invalid\r\n\r\n'").status_code == 400
class TestHTTPSNoUpstream(tservers.HTTPProxTest, CommonMixin):
ssl = True
no_upstream_cert = True
def test_cert_gen_error(self):
f = self.pathoc_raw()
f.connect((u"foo..bar".encode("utf8"), 0))
f.request("get:/")
assert "dummy cert" in "".join(self.proxy.log)
class TestHTTPSCertfile(tservers.HTTPProxTest, CommonMixin):
ssl = True
certfile = True