Web apps to addons

This commit removes the app registry, adds a wsgiapp addon base, and ports the
onboarding app to it.

mitmproxy/addons.py

@@ -12,6 +12,13 @@ class Addons:
         self.master = master
         master.options.changed.connect(self._options_update)
 
+    def clear(self):
+        """
+            Remove all addons.
+        """
+        self.done()
+        self.chain = []
+
     def get(self, name):
         """
             Retrieve an addon by name. Addon names are equal to the .name
@@ -43,8 +50,6 @@ class Addons:
         """
             Add addons to the end of the chain, and run their startup events.
         """
-        if not addons:
-            raise ValueError("No addons specified.")
         self.chain.extend(addons)
         for i in addons:
             self.startup(i)

mitmproxy/builtins/__init__.py

@@ -2,6 +2,7 @@ from mitmproxy.builtins import anticache
 from mitmproxy.builtins import anticomp
 from mitmproxy.builtins import clientplayback
 from mitmproxy.builtins import filestreamer
+from mitmproxy.builtins import onboarding
 from mitmproxy.builtins import replace
 from mitmproxy.builtins import script
 from mitmproxy.builtins import setheaders
@@ -13,6 +14,7 @@ from mitmproxy.builtins import streambodies
 
 def default_addons():
     return [
+        onboarding.Onboarding(),
         anticache.AntiCache(),
         anticomp.AntiComp(),
         stickyauth.StickyAuth(),

mitmproxy/builtins/onboarding.py

@@ -0,0 +1,17 @@
+from mitmproxy.builtins import wsgiapp
+from mitmproxy.builtins.onboardingapp import app
+
+
+class Onboarding(wsgiapp.WSGIApp):
+    def __init__(self):
+        super().__init__(app.Adapter(app.application), None, None)
+        self.enabled = False
+
+    def configure(self, options, updated):
+        self.host = options.app_host
+        self.port = options.app_port
+        self.enabled = options.app
+
+    def request(self, f):
+        if self.enabled:
+            super().request(f)

mitmproxy/builtins/onboarding/app.py

@@ -7,7 +7,7 @@ import tornado.wsgi
 from mitmproxy import utils
 from mitmproxy.proxy import config
 
-loader = tornado.template.Loader(utils.pkg_data.path("onboarding/templates"))
+loader = tornado.template.Loader(utils.pkg_data.path("builtins/onboardingapp/templates"))
 
 
 class Adapter(tornado.wsgi.WSGIAdapter):
@@ -91,4 +91,3 @@ application = tornado.web.Application(
     ],
     # debug=True
 )
-mapp = Adapter(application)

mitmproxy/builtins/onboardingapp/app.py

@@ -0,0 +1,109 @@
+import os
+
+import tornado.template
+import tornado.web
+import tornado.wsgi
+
+from mitmproxy import utils
+from mitmproxy.proxy import config
+from mitmproxy.builtins import wsgiapp
+
+loader = tornado.template.Loader(utils.pkg_data.path("builtins/onboardingapp/templates"))
+
+
+class Adapter(tornado.wsgi.WSGIAdapter):
+    # Tornado doesn't make the WSGI environment available to pages, so this
+    # hideous monkey patch is the easiest way to get to the mitmproxy.master
+    # variable.
+
+    def __init__(self, application):
+        self._application = application
+
+    def application(self, request):
+        request.master = self.environ["mitmproxy.master"]
+        return self._application(request)
+
+    def __call__(self, environ, start_response):
+        self.environ = environ
+        return tornado.wsgi.WSGIAdapter.__call__(
+            self,
+            environ,
+            start_response
+        )
+
+
+class Index(tornado.web.RequestHandler):
+
+    def get(self):
+        t = loader.load("index.html")
+        self.write(t.generate())
+
+
+class PEM(tornado.web.RequestHandler):
+
+    @property
+    def filename(self):
+        return config.CONF_BASENAME + "-ca-cert.pem"
+
+    def get(self):
+        p = os.path.join(self.request.master.options.cadir, self.filename)
+        p = os.path.expanduser(p)
+        self.set_header("Content-Type", "application/x-x509-ca-cert")
+        self.set_header(
+            "Content-Disposition",
+            "inline; filename={}".format(
+                self.filename))
+
+        with open(p, "rb") as f:
+            self.write(f.read())
+
+
+class P12(tornado.web.RequestHandler):
+
+    @property
+    def filename(self):
+        return config.CONF_BASENAME + "-ca-cert.p12"
+
+    def get(self):
+        p = os.path.join(self.request.master.options.cadir, self.filename)
+        p = os.path.expanduser(p)
+        self.set_header("Content-Type", "application/x-pkcs12")
+        self.set_header(
+            "Content-Disposition",
+            "inline; filename={}".format(
+                self.filename))
+
+        with open(p, "rb") as f:
+            self.write(f.read())
+
+
+application = tornado.web.Application(
+    [
+        (r"/", Index),
+        (r"/cert/pem", PEM),
+        (r"/cert/p12", P12),
+        (
+            r"/static/(.*)",
+            tornado.web.StaticFileHandler,
+            {
+                "path": utils.pkg_data.path("builtins/onboardingapp/static")
+            }
+        ),
+    ],
+    # debug=True
+)
+
+
+class Onboarding(wsgiapp.WSGIApp):
+    def __init__(self):
+        super().__init__(Adapter(application), None, None)
+        self.enabled = False
+
+    def configure(self, options, updated):
+        self.host = options.app_host
+        self.port = options.app_port
+        self.enabled = options.app
+
+    def request(self, f):
+        if self.enabled:
+            super().request(f)

mitmproxy/builtins/wsgiapp.py

@@ -0,0 +1,32 @@
+from mitmproxy import ctx
+
+from netlib import wsgi
+from netlib import version
+
+
+class WSGIApp:
+    def __init__(self, app, host, port):
+        self.app, self.host, self.port = app, host, port
+
+    def serve(self, app, flow):
+        """
+            Serves app on flow, and prevents further handling of the flow.
+        """
+        app = wsgi.WSGIAdaptor(
+            app,
+            flow.request.pretty_host,
+            flow.request.port,
+            version.MITMPROXY
+        )
+        err = app.serve(
+            flow,
+            flow.client_conn.wfile,
+            **{"mitmproxy.master": ctx.master}
+        )
+        if err:
+            ctx.log.warn("Error in wsgi app. %s" % err, "error")
+        flow.reply.kill()
+
+    def request(self, f):
+        if (f.request.pretty_host, f.request.port) == (self.host, self.port):
+            self.serve(self.app, f)

mitmproxy/console/master.py

@@ -246,9 +246,6 @@ class ConsoleMaster(flow.FlowMaster):
 
         self.view_stack = []
 
-        if options.app:
-            self.start_app(self.options.app_host, self.options.app_port)
-
         signals.call_in.connect(self.sig_call_in)
         signals.pop_view_state.connect(self.sig_pop_view_state)
         signals.replace_view_state.connect(self.sig_replace_view_state)

mitmproxy/dump.py

@@ -61,9 +61,6 @@ class DumpMaster(flow.FlowMaster):
                 self.add_log("Flow file corrupted.", "error")
                 raise DumpError(v)
 
-        if self.options.app:
-            self.start_app(self.options.app_host, self.options.app_port)
-
     def _readflow(self, paths):
         """
         Utitility function that reads a list of flows

mitmproxy/flow/__init__.py

@@ -1,13 +1,10 @@
-from mitmproxy.flow import export, modules
+from mitmproxy.flow import export
 from mitmproxy.flow.io import FlowWriter, FilteredFlowWriter, FlowReader, read_flows_from_paths
 from mitmproxy.flow.master import FlowMaster
-from mitmproxy.flow.modules import (
-    AppRegistry
-)
 from mitmproxy.flow.state import State, FlowView
 
 __all__ = [
-    "export", "modules",
+    "export",
     "FlowWriter", "FilteredFlowWriter", "FlowReader", "read_flows_from_paths",
-    "FlowMaster", "AppRegistry", "State", "FlowView",
+    "FlowMaster", "State", "FlowView",
 ]

mitmproxy/flow/master.py

@@ -1,15 +1,11 @@
 import os
 import sys
 
-from typing import Optional  # noqa
-
 from netlib import http
 from mitmproxy import controller
 from mitmproxy import exceptions
 from mitmproxy import models
 from mitmproxy.flow import io
-from mitmproxy.flow import modules
-from mitmproxy.onboarding import app
 from mitmproxy.protocol import http_replay
 
 
@@ -51,10 +47,6 @@ class FlowMaster(controller.Master):
         super().__init__(options)
         if server:
             self.add_server(server)
-        self.apps = modules.AppRegistry()
-
-    def start_app(self, host, port):
-        self.apps.add(app.mapp, host, port)
 
     def create_request(self, method, scheme, host, port, path):
         """
@@ -203,18 +195,7 @@ class FlowMaster(controller.Master):
 
     @controller.handler
     def request(self, f):
-        if f.live:
-            app = self.apps.get(f.request)
-            if app:
-                err = app.serve(
-                    f,
-                    f.client_conn.wfile,
-                    **{"mitmproxy.master": self}
-                )
-                if err:
-                    self.add_log("Error in wsgi app. %s" % err, "error")
-                f.reply.kill()
-                return
+        pass
 
     @controller.handler
     def responseheaders(self, f):

mitmproxy/flow/modules.py

@@ -1,29 +0,0 @@
-from netlib import wsgi
-from netlib import version
-
-
-class AppRegistry:
-    def __init__(self):
-        self.apps = {}
-
-    def add(self, app, domain, port):
-        """
-            Add a WSGI app to the registry, to be served for requests to the
-            specified domain, on the specified port.
-        """
-        self.apps[(domain, port)] = wsgi.WSGIAdaptor(
-            app,
-            domain,
-            port,
-            version.MITMPROXY
-        )
-
-    def get(self, request):
-        """
-            Returns an WSGIAdaptor instance if request matches an app, or None.
-        """
-        if (request.host, request.port) in self.apps:
-            return self.apps[(request.host, request.port)]
-        if "host" in request.headers:
-            host = request.headers["host"]
-            return self.apps.get((host, request.port), None)

mitmproxy/web/master.py

@@ -160,9 +160,6 @@ class WebMaster(flow.FlowMaster):
                 print("Stream file error: {}".format(err), file=sys.stderr)
                 sys.exit(1)
 
-        if self.options.app:
-            self.start_app(self.options.app_host, self.options.app_port)
-
     def run(self):  # pragma: no cover
 
         iol = tornado.ioloop.IOLoop.instance()

test/mitmproxy/builtins/test_onboarding.py

@@ -1,7 +1,10 @@
-from . import tservers
+from mitmproxy.builtins import onboarding
+from .. import tservers
 
 
 class TestApp(tservers.HTTPProxyTest):
+    def addons(self):
+        return [onboarding.Onboarding()]
 
     def test_basic(self):
         assert self.app("/").status_code == 200

test/mitmproxy/builtins/test_wsgiapp.py

@@ -0,0 +1,61 @@
+import flask
+
+from .. import tservers
+from mitmproxy.builtins import wsgiapp
+
+testapp = flask.Flask(__name__)
+
+
+@testapp.route("/")
+def hello():
+    return "testapp"
+
+
+@testapp.route("/error")
+def error():
+    raise ValueError("An exception...")
+
+
+def errapp(environ, start_response):
+    raise ValueError("errapp")
+
+
+class TestApp(tservers.HTTPProxyTest):
+    def addons(self):
+        return [
+            wsgiapp.WSGIApp(testapp, "testapp", 80),
+            wsgiapp.WSGIApp(errapp, "errapp", 80)
+        ]
+
+    def test_simple(self):
+        p = self.pathoc()
+        with p.connect():
+            ret = p.request("get:'http://testapp/'")
+        assert ret.status_code == 200
+
+    def _test_app_err(self):
+        p = self.pathoc()
+        with p.connect():
+            ret = p.request("get:'http://errapp/'")
+        assert ret.status_code == 500
+        assert b"ValueError" in ret.content
+
+
+def _test_app_registry():
+    ar = flow.AppRegistry()
+    ar.add("foo", "domain", 80)
+
+    r = HTTPRequest.wrap(netlib.tutils.treq())
+    r.host = "domain"
+    r.port = 80
+    assert ar.get(r)
+
+    r.port = 81
+    assert not ar.get(r)
+
+    r = HTTPRequest.wrap(netlib.tutils.treq())
+    r.host = "domain2"
+    r.port = 80
+    assert not ar.get(r)
+    r.headers["host"] = "domain"
+    assert ar.get(r)

test/mitmproxy/protocol/test_http2.py

@@ -96,7 +96,6 @@ class _Http2TestBase:
         cls.config = ProxyConfig(opts)
 
         tmaster = tservers.TestMaster(opts, cls.config)
-        tmaster.start_app(options.APP_HOST, options.APP_PORT)
         cls.proxy = tservers.ProxyThread(tmaster)
         cls.proxy.start()
 
@@ -119,8 +118,7 @@ class _Http2TestBase:
         return self.proxy.tmaster
 
     def setup(self):
-        self.master.clear_log()
-        self.master.state.clear()
+        self.master.reset([])
         self.server.server.handle_server_event = self.handle_server_event
 
     def _setup_connection(self):

test/mitmproxy/protocol/test_websockets.py

@@ -50,7 +50,6 @@ class _WebSocketsTestBase:
         cls.config = ProxyConfig(opts)
 
         tmaster = tservers.TestMaster(opts, cls.config)
-        tmaster.start_app(options.APP_HOST, options.APP_PORT)
         cls.proxy = tservers.ProxyThread(tmaster)
         cls.proxy.start()
 
@@ -74,8 +73,7 @@ class _WebSocketsTestBase:
         return self.proxy.tmaster
 
     def setup(self):
-        self.master.clear_log()
-        self.master.state.clear()
+        self.master.reset([])
         self.server.server.handle_websockets = self.handle_websockets
 
     def _setup_connection(self):

test/mitmproxy/test_addons.py

@@ -18,3 +18,5 @@ def test_simple():
     a.add(TAddon("one"))
     assert a.get("one")
     assert not a.get("two")
+    a.clear()
+    assert not a.chain

test/mitmproxy/test_dump.py

@@ -99,11 +99,6 @@ class TestDumpMaster(mastertest.MasterTest):
             self.mkmaster("~u foo", verbosity=1), 1, b""
         )
 
-    def test_app(self):
-        o = dump.Options(app=True)
-        m = dump.DumpMaster(None, o)
-        assert len(m.apps.apps) == 1
-
     def test_replacements(self):
         o = dump.Options(
             replacements=[(".*", "content", "foo")],

test/mitmproxy/test_flow.py

@@ -17,26 +17,6 @@ from mitmproxy.models.connections import ClientConnection
 from . import tutils
 
 
-def test_app_registry():
-    ar = flow.AppRegistry()
-    ar.add("foo", "domain", 80)
-
-    r = HTTPRequest.wrap(netlib.tutils.treq())
-    r.host = "domain"
-    r.port = 80
-    assert ar.get(r)
-
-    r.port = 81
-    assert not ar.get(r)
-
-    r = HTTPRequest.wrap(netlib.tutils.treq())
-    r.host = "domain2"
-    r.port = 80
-    assert not ar.get(r)
-    r.headers["host"] = "domain"
-    assert ar.get(r)
-
-
 class TestHTTPFlow:
 
     def test_copy(self):

test/mitmproxy/test_server.py

@@ -201,23 +201,7 @@ class TcpMixin:
         # assert any("306" in m for m in self.master.tlog)
 
 
-class AppMixin:
-
-    def test_app(self):
-        ret = self.app("/")
-        assert ret.status_code == 200
-        assert b"mitmproxy" in ret.content
-
-
-class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin):
-
-    def test_app_err(self):
-        p = self.pathoc()
-        with p.connect():
-            ret = p.request("get:'http://errapp/'")
-        assert ret.status_code == 500
-        assert b"ValueError" in ret.content
-
+class TestHTTP(tservers.HTTPProxyTest, CommonMixin):
     def test_invalid_connect(self):
         t = tcp.TCPClient(("127.0.0.1", self.proxy.port))
         with t.connect():
@@ -897,7 +881,7 @@ class TestIncompleteResponse(tservers.HTTPProxyTest):
         assert self.pathod("200").status_code == 502
 
 
-class TestUpstreamProxy(tservers.HTTPUpstreamProxyTest, CommonMixin, AppMixin):
+class TestUpstreamProxy(tservers.HTTPUpstreamProxyTest, CommonMixin):
     ssl = False
 
 

test/mitmproxy/tservers.py

@@ -1,7 +1,6 @@
 import os.path
 import threading
 import tempfile
-import flask
 import mock
 import sys
 
@@ -13,37 +12,26 @@ import pathod.pathoc
 from mitmproxy import flow, controller, options
 import netlib.exceptions
 
-testapp = flask.Flask(__name__)
-
-
-@testapp.route("/")
-def hello():
-    return "testapp"
-
-
-@testapp.route("/error")
-def error():
-    raise ValueError("An exception...")
-
-
-def errapp(environ, start_response):
-    raise ValueError("errapp")
-
 
 class TestMaster(flow.FlowMaster):
 
     def __init__(self, opts, config):
         s = ProxyServer(config)
         flow.FlowMaster.__init__(self, opts, s)
+
+    def clear_addons(self, addons):
+        self.addons.clear()
         self.state = state.State()
         self.addons.add(self.state)
-        self.apps.add(testapp, "testapp", 80)
-        self.apps.add(errapp, "errapp", 80)
-        self.clear_log()
+        self.addons.add(*addons)
 
     def clear_log(self):
         self.tlog = []
 
+    def reset(self, addons):
+        self.clear_addons(addons)
+        self.clear_log()
+
     @controller.handler
     def log(self, e):
         self.tlog.append(e.msg)
@@ -94,7 +82,6 @@ class ProxyTestBase:
         opts = cls.get_options()
         cls.config = ProxyConfig(opts)
         tmaster = cls.masterclass(opts, cls.config)
-        tmaster.start_app(options.APP_HOST, options.APP_PORT)
         cls.proxy = ProxyThread(tmaster)
         cls.proxy.start()
 
@@ -116,8 +103,7 @@ class ProxyTestBase:
                 raise
 
     def setup(self):
-        self.master.state.clear()
-        self.master.clear_log()
+        self.master.reset(self.addons())
         self.server.clear_log()
         self.server2.clear_log()
 
@@ -135,6 +121,12 @@ class ProxyTestBase:
             ssl_insecure=True,
         )
 
+    def addons(self):
+        """
+            Can be over-ridden to add a standard set of addons to tests.
+        """
+        return []
+
 
 class LazyPathoc(pathod.pathoc.Pathoc):
     def __init__(self, lazy_connect, *args, **kwargs):
@@ -330,8 +322,7 @@ class ChainProxyTest(ProxyTestBase):
     def setup(self):
         super().setup()
         for proxy in self.chain:
-            proxy.tmaster.clear_log()
-            proxy.tmaster.state.clear()
+            proxy.tmaster.reset(self.addons())
 
     @classmethod
     def get_options(cls):