Merge remote-tracking branch 'origin/master'

doc-src/scripting/inlinescripts.html

@@ -25,7 +25,7 @@ The new header will be added to all responses passing through the proxy.
 
 ## Events
 
-### start(ScriptContext)
+### start(ScriptContext, argv)
 
 Called once on startup, before any other events.
 

doc-src/transparent/osx.html

@@ -67,3 +67,15 @@ rdr on en2 inet proto tcp to any port 443 -> 127.0.0.1 port 8080
 
 
 </ol>
+
+Note that the **rdr** rules in the pf.conf given above only apply to inbound
+traffic. This means that they will NOT redirect traffic coming from the box
+running pf itself. We can't distinguish between an outbound connection from a
+non-mitmproxy app, and an outbound connection from mitmproxy itself - if you
+want to intercept your OSX traffic, you should use an external host to run
+mitmproxy. None the less, pf is flexible to cater for a range of creative
+possibilities, like intercepting traffic emanating from VMs.  See the
+**pf.conf** man page for more.
+
+
+

examples/stub.py

@@ -1,8 +1,7 @@
 """
     This is a script stub, with definitions for all events.
 """
-
+def start(ctx, argv):
-def start(ctx):
     """
         Called once on script startup, before any other events.
     """

libmproxy/cmdline.py

@@ -1,6 +1,8 @@
 import proxy
 import re, filt
 import argparse
+import shlex
+import os
 
 APP_DOMAIN = "mitm"
 APP_IP = "1.1.1.1"
@@ -143,7 +145,7 @@ def get_common_options(options):
         replacements = reps,
         setheaders = setheaders,
         server_replay = options.server_replay,
-        script = options.script,
+        scripts = options.scripts,
         stickycookie = stickycookie,
         stickyauth = stickyauth,
         showhost = options.showhost,
@@ -189,6 +191,11 @@ def common_options(parser):
         action="store", dest="reverse_proxy", default=None,
         help="Reverse proxy to upstream server: http[s]://host[:port]"
     )
+    parser.add_argument(
+        "-F",
+        action="store", dest="forward_proxy", default=None,
+        help="Proxy to unconditionally forward to: http[s]://host[:port]"
+    )
     parser.add_argument(
         "-q",
         action="store_true", dest="quiet",
@@ -201,8 +208,9 @@ def common_options(parser):
     )
     parser.add_argument(
         "-s",
-        action="store", dest="script", default=None,
+        action="append", type=lambda x: shlex.split(x,posix=(os.name != "nt")), dest="scripts", default=[],
-        help="Run a script."
+        metavar='"script.py --bar"',
+        help="Run a script. Surround with quotes to pass script arguments. Can be passed multiple times."
     )
     parser.add_argument(
         "-t",

libmproxy/console/__init__.py

@@ -459,7 +459,7 @@ class ConsoleMaster(flow.FlowMaster):
             self._run_script_method("response", s, f)
         if f.error:
             self._run_script_method("error", s, f)
-        s.run("done")
+        s.unload()
         self.refresh_flow(f)
         self.state.last_script = path
 

libmproxy/console/common.py

@@ -190,7 +190,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
 
         delta = f.response.timestamp_end - f.response.timestamp_start
         size = len(f.response.content) + f.response.get_header_size()
-        rate = utils.pretty_size(size / delta)
+        rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) )
 
         d.update(dict(
             resp_code = f.response.code,

libmproxy/dump.py

@@ -24,7 +24,7 @@ class Options(object):
         "rheaders",
         "setheaders",
         "server_replay",
-        "script",
+        "scripts",
         "showhost",
         "stickycookie",
         "stickyauth",
@@ -109,8 +109,8 @@ class DumpMaster(flow.FlowMaster):
                 not options.keepserving
             )
 
-        if options.script:
+        for script_argv in options.scripts:
-            err = self.load_script(options.script)
+            err = self.load_script(script_argv)
             if err:
                 raise DumpError(err)
 
@@ -221,8 +221,8 @@ class DumpMaster(flow.FlowMaster):
 
     def run(self):  # pragma: no cover
         if self.o.rfile and not self.o.keepserving:
-            if self.script:
+            for script in self.scripts:
-                self.load_script(None)
+                self.unload_script(script)
             return
         try:
             return flow.FlowMaster.run(self)

libmproxy/flow.py

@@ -1349,7 +1349,7 @@ class FlowMaster(controller.Master):
         self.server_playback = None
         self.client_playback = None
         self.kill_nonreplay = False
-        self.script = None
+        self.scripts = []
         self.pause_scripts = False
 
         self.stickycookie_state = False
@@ -1385,36 +1385,42 @@ class FlowMaster(controller.Master):
         """
         pass
 
-    def get_script(self, path):
+    def get_script(self, script_argv):
         """
             Returns an (error, script) tuple.
         """
-        s = script.Script(path, ScriptContext(self))
+        s = script.Script(script_argv, ScriptContext(self))
         try:
             s.load()
         except script.ScriptError, v:
             return (v.args[0], None)
-        ret = s.run("start")
-        if not ret[0] and ret[1]:
-            return ("Error in script start:\n\n" + ret[1][1], None)
         return (None, s)
 
-    def load_script(self, path):
+    def unload_script(self,script):
+        script.unload()
+        self.scripts.remove(script)
+
+    def load_script(self, script_argv):
         """
             Loads a script. Returns an error description if something went
-            wrong. If path is None, the current script is terminated.
+            wrong.
         """
-        if path is None:
+        r = self.get_script(script_argv)
-            self.run_script_hook("done")
-            self.script = None
-        else:
-            r = self.get_script(path)
         if r[0]:
             return r[0]
         else:
-                if self.script:
+            self.scripts.append(r[1])
-                    self.run_script_hook("done")
+
-                self.script = r[1]
+    def run_single_script_hook(self, script, name, *args, **kwargs):
+        if script and not self.pause_scripts:
+            ret = script.run(name, *args, **kwargs)
+            if not ret[0] and ret[1]:
+                e = "Script error:\n" + ret[1][1]
+                self.add_event(e, "error")
+
+    def run_script_hook(self, name, *args, **kwargs):
+        for script in self.scripts:
+            self.run_single_script_hook(script, name, *args, **kwargs)
       
     def set_stickycookie(self, txt):
         if txt:
@@ -1565,13 +1571,6 @@ class FlowMaster(controller.Master):
             if block:
                 rt.join()
 
-    def run_script_hook(self, name, *args, **kwargs):
-        if self.script and not self.pause_scripts:
-            ret = self.script.run(name, *args, **kwargs)
-            if not ret[0] and ret[1]:
-                e = "Script error:\n" + ret[1][1]
-                self.add_event(e, "error")
-
     def handle_clientconnect(self, cc):
         self.run_script_hook("clientconnect", cc)
         cc.reply()
@@ -1580,6 +1579,13 @@ class FlowMaster(controller.Master):
         self.run_script_hook("clientdisconnect", r)
         r.reply()
 
+    def handle_serverconnection(self, sc):
+        # To unify the mitmproxy script API, we call the script hook "serverconnect" rather than "serverconnection".
+        # As things are handled differently in libmproxy (ClientConnect + ClientDisconnect vs ServerConnection class),
+        # there is no "serverdisonnect" event at the moment.
+        self.run_script_hook("serverconnect", sc)
+        sc.reply()
+
     def handle_error(self, r):
         f = self.state.add_error(r)
         if f:
@@ -1613,8 +1619,8 @@ class FlowMaster(controller.Master):
         return f
 
     def shutdown(self):
-        if self.script:
+        for script in self.scripts:
-            self.load_script(None)
+            self.unload_script(script)
         controller.Master.shutdown(self)
         if self.stream:
             for i in self.state._flow_list:

libmproxy/proxy.py

@@ -23,13 +23,14 @@ class Log:
 
 
 class ProxyConfig:
-    def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, transparent_proxy=None, authenticator=None):
+    def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, forward_proxy=None, transparent_proxy=None, authenticator=None):
         self.certfile = certfile
         self.cacert = cacert
         self.clientcerts = clientcerts
         self.no_upstream_cert = no_upstream_cert
         self.body_size_limit = body_size_limit
         self.reverse_proxy = reverse_proxy
+        self.forward_proxy = forward_proxy
         self.transparent_proxy = transparent_proxy
         self.authenticator = authenticator
         self.certstore = certutils.CertStore()
@@ -158,6 +159,7 @@ class ProxyHandler(tcp.BaseHandler):
         if not self.server_conn:
             try:
                 self.server_conn = ServerConnection(self.config, scheme, host, port, sni)
+                self.channel.ask(self.server_conn)
                 self.server_conn.connect()
             except tcp.NetLibError, v:
                 raise ProxyError(502, v)
@@ -219,7 +221,12 @@ class ProxyHandler(tcp.BaseHandler):
                     # the case, we want to reconnect without sending an error
                     # to the client.
                     while 1:
+                        if self.config.forward_proxy:
+                            forward_scheme, forward_host, forward_port = self.config.forward_proxy
+                            sc = self.get_server_connection(cc, forward_scheme, forward_host, forward_port, self.sni)
+                        else:
                             sc = self.get_server_connection(cc, scheme, host, port, self.sni)
+                            
                         sc.send(request)
                         if sc.requestcount == 1: # add timestamps only for first request (others are not directly affected)
                             request.tcp_setup_timestamp = sc.tcp_setup_timestamp
@@ -258,13 +265,13 @@ class ProxyHandler(tcp.BaseHandler):
                 else:
                     response = response_reply
                     self.send_response(response)
-                    if request and http.request_connection_close(request.httpversion, request.headers):
+                    if request and http.connection_close(request.httpversion, request.headers):
                         return
                     # We could keep the client connection when the server
                     # connection needs to go away.  However, we want to mimic
                     # behaviour as closely as possible to the client, so we
                     # disconnect.
-                    if http.response_connection_close(response.httpversion, response.headers):
+                    if http.connection_close(response.httpversion, response.headers):
                         return
         except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e:
             if hasattr(e, "code"):
@@ -310,6 +317,17 @@ class ProxyHandler(tcp.BaseHandler):
                 raise ProxyError(502, "Unable to generate dummy cert.")
             return ret
 
+    def establish_ssl(self, client_conn, host, port):
+        dummycert = self.find_cert(client_conn, host, port, host)
+        sni = HandleSNI(
+            self, client_conn, host, port,
+            dummycert, self.config.certfile or self.config.cacert
+        )
+        try:
+            self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
+        except tcp.NetLibError, v:
+            raise ProxyError(400, str(v))
+
     def get_line(self, fp):
         """
             Get a line, possibly preceded by a blank.
@@ -329,15 +347,7 @@ class ProxyHandler(tcp.BaseHandler):
         if port in self.config.transparent_proxy["sslports"]:
             scheme = "https"
             if not self.ssl_established:
-                dummycert = self.find_cert(client_conn, host, port, host)
+                self.establish_ssl(client_conn, host, port)
-                sni = HandleSNI(
-                    self, client_conn, host, port,
-                    dummycert, self.config.certfile or self.config.cacert
-                )
-                try:
-                    self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
-                except tcp.NetLibError, v:
-                    raise ProxyError(400, str(v))
         else:
             scheme = "http"
         line = self.get_line(self.rfile)
@@ -372,15 +382,7 @@ class ProxyHandler(tcp.BaseHandler):
                             '\r\n'
                             )
                 self.wfile.flush()
-                dummycert = self.find_cert(client_conn, host, port, host)
+                self.establish_ssl(client_conn, host, port)
-                sni = HandleSNI(
-                    self, client_conn, host, port,
-                    dummycert, self.config.certfile or self.config.cacert
-                )
-                try:
-                    self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
-                except tcp.NetLibError, v:
-                    raise ProxyError(400, str(v))
                 self.proxy_connect_state = (host, port, httpversion)
                 line = self.rfile.readline(line)
 
@@ -414,10 +416,12 @@ class ProxyHandler(tcp.BaseHandler):
                     )
 
     def read_request_reverse(self, client_conn):
+        scheme, host, port = self.config.reverse_proxy
+        if scheme.lower() == "https" and not self.ssl_established:
+            self.establish_ssl(client_conn, host, port)
         line = self.get_line(self.rfile)
         if line == "":
             return None
-        scheme, host, port = self.config.reverse_proxy
         r = http.parse_init_http(line)
         if not r:
             raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
@@ -427,7 +431,7 @@ class ProxyHandler(tcp.BaseHandler):
                     self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                 )
         return flow.Request(
-                    client_conn, httpversion, host, port, "http", method, path, headers, content,
+                    client_conn, httpversion, host, port, scheme, method, path, headers, content,
                     self.rfile.first_byte_timestamp, utils.timestamp()
                )
 
@@ -594,6 +598,13 @@ def process_proxy_options(parser, options):
     else:
         rp = None
 
+    if options.forward_proxy:
+        fp = utils.parse_proxy_spec(options.forward_proxy)
+        if not fp:
+            return parser.error("Invalid forward proxy specification: %s"%options.forward_proxy)
+    else:
+        fp = None
+
     if options.clientcerts:
         options.clientcerts = os.path.expanduser(options.clientcerts)
         if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts):
@@ -623,6 +634,7 @@ def process_proxy_options(parser, options):
         body_size_limit = body_size_limit,
         no_upstream_cert = options.no_upstream_cert,
         reverse_proxy = rp,
+        forward_proxy = fp,
         transparent_proxy = trans,
         authenticator = authenticator
     )

libmproxy/script.py

@@ -8,12 +8,12 @@ class Script:
     """
         The instantiator should do something along this vein:
 
-            s = Script(path, master)
+            s = Script(argv, master)
             s.load()
-            s.run("start")
     """
-    def __init__(self, path, ctx):
+    def __init__(self, argv, ctx):
-        self.path, self.ctx = path, ctx
+        self.argv = argv
+        self.ctx = ctx
         self.ns = None
 
     def load(self):
@@ -23,17 +23,21 @@ class Script:
             Raises ScriptError on failure, with argument equal to an error
             message that may be a formatted traceback.
         """
-        path = os.path.expanduser(self.path)
+        path = os.path.expanduser(self.argv[0])
         if not os.path.exists(path):
-            raise ScriptError("No such file: %s"%self.path)
+            raise ScriptError("No such file: %s" % path)
         if not os.path.isfile(path):
-            raise ScriptError("Not a file: %s"%self.path)
+            raise ScriptError("Not a file: %s" % path)
         ns = {}
         try:
             execfile(path, ns, ns)
+            self.ns = ns
+            self.run("start", self.argv)
         except Exception, v:
             raise ScriptError(traceback.format_exc(v))
-        self.ns = ns
+
+    def unload(self):
+        return self.run("done")
 
     def run(self, name, *args, **kwargs):
         """

test/scripts/a.py

@@ -1,5 +1,13 @@
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--var', type=int)
 
 var = 0
+def start(ctx, argv):
+    global var
+    var = parser.parse_args(argv[1:]).var
+
 def here(ctx):
     global var
     var += 1

test/scripts/starterr.py

@@ -1,3 +1,3 @@
 
-def start(ctx):
+def start(ctx, argv):
     raise ValueError

test/test_cmdline.py

@@ -1,6 +1,7 @@
 import argparse
 from libmproxy import cmdline
 import tutils
+import os.path
 
 
 def test_parse_replace_hook():
@@ -39,6 +40,18 @@ def test_parse_setheaders():
     x = cmdline.parse_setheader("/foo/bar/voing")
     assert x == ("foo", "bar", "voing")
 
+def test_shlex():
+    """
+    shlex.split assumes posix=True by default, we do manual detection for windows.
+    Test whether script paths are parsed correctly
+    """
+    absfilepath = os.path.normcase(os.path.abspath(__file__))
+
+    parser = argparse.ArgumentParser()
+    cmdline.common_options(parser)
+    opts = parser.parse_args(args=["-s",absfilepath])
+    
+    assert os.path.isfile(opts.scripts[0][0])
 
 def test_common():
     parser = argparse.ArgumentParser()

test/test_flow.py

@@ -544,9 +544,11 @@ class TestFlowMaster:
         fm = flow.FlowMaster(None, s)
         assert not fm.load_script(tutils.test_data.path("scripts/a.py"))
         assert not fm.load_script(tutils.test_data.path("scripts/a.py"))
-        assert not fm.load_script(None)
+        assert not fm.unload_script(fm.scripts[0])
+        assert not fm.unload_script(fm.scripts[0])
         assert fm.load_script("nonexistent")
         assert "ValueError" in fm.load_script(tutils.test_data.path("scripts/starterr.py"))
+        assert len(fm.scripts) == 0
 
     def test_replay(self):
         s = flow.State()
@@ -572,20 +574,27 @@ class TestFlowMaster:
         assert not fm.load_script(tutils.test_data.path("scripts/all.py"))
         req = tutils.treq()
         fm.handle_clientconnect(req.client_conn)
-        assert fm.script.ns["log"][-1] == "clientconnect"
+        assert fm.scripts[0].ns["log"][-1] == "clientconnect"
         f = fm.handle_request(req)
-        assert fm.script.ns["log"][-1] == "request"
+        assert fm.scripts[0].ns["log"][-1] == "request"
         resp = tutils.tresp(req)
         fm.handle_response(resp)
-        assert fm.script.ns["log"][-1] == "response"
+        assert fm.scripts[0].ns["log"][-1] == "response"
+        #load second script
+        assert not fm.load_script(tutils.test_data.path("scripts/all.py"))
+        assert len(fm.scripts) == 2
         dc = flow.ClientDisconnect(req.client_conn)
         dc.reply = controller.DummyReply()
         fm.handle_clientdisconnect(dc)
-        assert fm.script.ns["log"][-1] == "clientdisconnect"
+        assert fm.scripts[0].ns["log"][-1] == "clientdisconnect"
+        assert fm.scripts[1].ns["log"][-1] == "clientdisconnect"
+        #unload first script
+        fm.unload_script(fm.scripts[0])
+        assert len(fm.scripts) == 1
         err = flow.Error(f.request, "msg")
         err.reply = controller.DummyReply()
         fm.handle_error(err)
-        assert fm.script.ns["log"][-1] == "error"
+        assert fm.scripts[0].ns["log"][-1] == "error"
 
     def test_duplicate_flow(self):
         s = flow.State()

test/test_script.py

@@ -1,5 +1,7 @@
 from libmproxy import script, flow
 import tutils
+import shlex
+import os
 
 class TestScript:
     def test_simple(self):
@@ -7,11 +9,12 @@ class TestScript:
         fm = flow.FlowMaster(None, s)
         ctx = flow.ScriptContext(fm)
 
-        p = script.Script(tutils.test_data.path("scripts/a.py"), ctx)
+        p = script.Script(shlex.split(tutils.test_data.path("scripts/a.py")+" --var 40", posix=(os.name != "nt")), ctx)
         p.load()
+
         assert "here" in p.ns
-        assert p.run("here") == (True, 1)
+        assert p.run("here") == (True, 41)
-        assert p.run("here") == (True, 2)
+        assert p.run("here") == (True, 42)
 
         ret = p.run("errargs")
         assert not ret[0]
@@ -19,12 +22,12 @@ class TestScript:
 
         # Check reload
         p.load()
-        assert p.run("here") == (True, 1)
+        assert p.run("here") == (True, 41)
 
     def test_duplicate_flow(self):
         s = flow.State()
         fm = flow.FlowMaster(None, s)
-        fm.load_script(tutils.test_data.path("scripts/duplicate_flow.py"))
+        fm.load_script([tutils.test_data.path("scripts/duplicate_flow.py")])
         r = tutils.treq()
         fm.handle_request(r)
         assert fm.state.flow_count() == 2
@@ -37,25 +40,25 @@ class TestScript:
         ctx = flow.ScriptContext(fm)
 
 
-        s = script.Script("nonexistent", ctx)
+        s = script.Script(["nonexistent"], ctx)
         tutils.raises(
             "no such file",
             s.load
         )
 
-        s = script.Script(tutils.test_data.path("scripts"), ctx)
+        s = script.Script([tutils.test_data.path("scripts")], ctx)
         tutils.raises(
             "not a file",
             s.load
         )
 
-        s = script.Script(tutils.test_data.path("scripts/syntaxerr.py"), ctx)
+        s = script.Script([tutils.test_data.path("scripts/syntaxerr.py")], ctx)
         tutils.raises(
             script.ScriptError,
             s.load
         )
 
-        s = script.Script(tutils.test_data.path("scripts/loaderr.py"), ctx)
+        s = script.Script([tutils.test_data.path("scripts/loaderr.py")], ctx)
         tutils.raises(
             script.ScriptError,
             s.load

test/test_server.py

@@ -191,16 +191,6 @@ class TestHTTPS(tservers.HTTPProxTest, CommonMixin):
         assert p.request("get:/:i0,'invalid\r\n\r\n'").status_code == 400
 
 
-class TestHTTPSNoUpstream(tservers.HTTPProxTest, CommonMixin):
-    ssl = True
-    no_upstream_cert = True
-    def test_cert_gen_error(self):
-        f = self.pathoc_raw()
-        f.connect((u"foo..bar".encode("utf8"), 0))
-        f.request("get:/")
-        assert "dummy cert" in "".join(self.proxy.log)
-
-
 class TestHTTPSCertfile(tservers.HTTPProxTest, CommonMixin):
     ssl = True
     certfile = True