Merge pull request #1630 from cortesi/multiserv

Let's dispense with this fiction that we support multiple servers

mitmproxy/console/master.py

@@ -221,7 +221,7 @@ class Options(mitmproxy.options.Options):
 class ConsoleMaster(flow.FlowMaster):
     palette = []
 
-    def __init__(self, server, options):
+    def __init__(self, options, server):
         flow.FlowMaster.__init__(self, options, server)
         self.state = ConsoleState()
         self.stream_path = None

mitmproxy/controller.py

@@ -87,14 +87,14 @@ class Master:
     """
         The master handles mitmproxy's main event loop.
     """
-    def __init__(self, opts, *servers):
+    def __init__(self, opts, server):
         self.options = opts or options.Options()
         self.addons = addons.Addons(self)
         self.event_queue = queue.Queue()
         self.should_exit = threading.Event()
-        self.servers = []
+        self.server = server
-        for i in servers:
+        channel = Channel(self.event_queue, self.should_exit)
-            self.add_server(i)
+        server.set_channel(channel)
 
     @contextlib.contextmanager
     def handlecontext(self):
@@ -121,16 +121,9 @@ class Master:
         with self.handlecontext():
             self.addons("log", LogEntry(e, level))
 
-    def add_server(self, server):
-        # We give a Channel to the server which can be used to communicate with the master
-        channel = Channel(self.event_queue, self.should_exit)
-        server.set_channel(channel)
-        self.servers.append(server)
-
     def start(self):
         self.should_exit.clear()
-        for server in self.servers:
+        ServerThread(self.server).start()
-            ServerThread(server).start()
 
     def run(self):
         self.start()
@@ -168,8 +161,7 @@ class Master:
         return changed
 
     def shutdown(self):
-        for server in self.servers:
+        self.server.shutdown()
-            server.shutdown()
         self.should_exit.set()
         self.addons.done()
 

mitmproxy/dump.py

@@ -32,7 +32,7 @@ class Options(options.Options):
 
 class DumpMaster(flow.FlowMaster):
 
-    def __init__(self, server, options):
+    def __init__(self, options, server):
         flow.FlowMaster.__init__(self, options, server)
         self.has_errored = False
         self.addons.add(termlog.TermLog())
@@ -41,7 +41,7 @@ class DumpMaster(flow.FlowMaster):
         # This line is just for type hinting
         self.options = self.options  # type: Options
 
-        if not self.options.no_server and server:
+        if not self.options.no_server:
             self.add_log(
                 "Proxy server listening at http://{}".format(server.address),
                 "info"

mitmproxy/flow/master.py

@@ -35,19 +35,6 @@ def event_sequence(f):
 
 
 class FlowMaster(controller.Master):
-
-    @property
-    def server(self):
-        # At some point, we may want to have support for multiple servers.
-        # For now, this suffices.
-        if len(self.servers) > 0:
-            return self.servers[0]
-
-    def __init__(self, options, server):
-        super().__init__(options)
-        if server:
-            self.add_server(server)
-
     def create_request(self, method, scheme, host, port, path):
         """
             this method creates a new artificial and minimalist request also adds it to flowlist

mitmproxy/main.py

@@ -95,7 +95,7 @@ def mitmdump(args=None):  # pragma: no cover
         dump_options.keepserving = args.keepserving
         dump_options.filtstr = " ".join(args.args) if args.args else None
         server = process_options(parser, dump_options, args)
-        master = dump.DumpMaster(server, dump_options)
+        master = dump.DumpMaster(dump_options, server)
 
         def cleankill(*args, **kwargs):
             master.shutdown()
@@ -132,7 +132,7 @@ def mitmweb(args=None):  # pragma: no cover
         web_options.process_web_options(parser)
 
         server = process_options(parser, web_options, args)
-        m = web.master.WebMaster(server, web_options)
+        m = web.master.WebMaster(web_options, server)
     except exceptions.OptionsError as e:
         print("mitmweb: %s" % e, file=sys.stderr)
         sys.exit(1)

mitmproxy/proxy/server.py

@@ -15,8 +15,9 @@ from netlib.http import http1
 class DummyServer:
     bound = False
 
-    def __init__(self, config):
+    def __init__(self, config=None):
         self.config = config
+        self.address = "dummy"
 
     def set_channel(self, channel):
         pass

mitmproxy/web/master.py

@@ -132,7 +132,7 @@ class Options(options.Options):
 
 class WebMaster(flow.FlowMaster):
 
-    def __init__(self, server, options):
+    def __init__(self, options, server):
         super().__init__(options, server)
         self.state = WebState()
         self.addons.add(*builtins.default_addons())

test/mitmproxy/builtins/test_anticache.py

@@ -2,12 +2,13 @@ from .. import tutils, mastertest
 from mitmproxy.builtins import anticache
 from mitmproxy.flow import master
 from mitmproxy import options
+from mitmproxy import proxy
 
 
 class TestAntiCache(mastertest.MasterTest):
     def test_simple(self):
         o = options.Options(anticache = True)
-        m = master.FlowMaster(o, None)
+        m = master.FlowMaster(o, proxy.DummyServer())
         sa = anticache.AntiCache()
         m.addons.add(sa)
 

test/mitmproxy/builtins/test_anticomp.py

@@ -2,12 +2,13 @@ from .. import tutils, mastertest
 from mitmproxy.builtins import anticomp
 from mitmproxy.flow import master
 from mitmproxy import options
+from mitmproxy import proxy
 
 
 class TestAntiComp(mastertest.MasterTest):
     def test_simple(self):
         o = options.Options(anticomp = True)
-        m = master.FlowMaster(o, None)
+        m = master.FlowMaster(o, proxy.DummyServer())
         sa = anticomp.AntiComp()
         m.addons.add(sa)
 

test/mitmproxy/builtins/test_dumper.py

@@ -6,6 +6,7 @@ from mitmproxy.builtins import dumper
 from mitmproxy import exceptions
 from mitmproxy import dump
 from mitmproxy import models
+from mitmproxy import proxy
 import netlib.tutils
 import mock
 
@@ -76,7 +77,7 @@ class TestContentView(mastertest.MasterTest):
             verbosity=3,
             tfile=sio,
         )
-        m = mastertest.RecordingMaster(o, None)
+        m = mastertest.RecordingMaster(o, proxy.DummyServer())
         d = dumper.Dumper()
         m.addons.add(d)
         m.response(tutils.tflow())

test/mitmproxy/builtins/test_filestreamer.py

@@ -5,6 +5,7 @@ import os.path
 from mitmproxy.builtins import filestreamer
 from mitmproxy.flow import master, FlowReader
 from mitmproxy import options
+from mitmproxy import proxy
 
 
 class TestStream(mastertest.MasterTest):
@@ -19,7 +20,7 @@ class TestStream(mastertest.MasterTest):
             o = options.Options(
                 outfile = (p, "wb")
             )
-            m = master.FlowMaster(o, None)
+            m = master.FlowMaster(o, proxy.DummyServer())
             sa = filestreamer.FileStreamer()
 
             m.addons.add(sa)

test/mitmproxy/builtins/test_replace.py

@@ -2,6 +2,7 @@ from .. import tutils, mastertest, tservers
 from mitmproxy.builtins import replace
 from mitmproxy.flow import master
 from mitmproxy import options
+from mitmproxy import proxy
 
 
 class TestReplace(mastertest.MasterTest):
@@ -35,7 +36,7 @@ class TestReplace(mastertest.MasterTest):
                 ("~s", "foo", "bar"),
             ]
         )
-        m = master.FlowMaster(o, None)
+        m = master.FlowMaster(o, proxy.DummyServer())
         sa = replace.Replace()
         m.addons.add(sa)
 

test/mitmproxy/builtins/test_script.py

@@ -6,6 +6,7 @@ import time
 import re
 from mitmproxy import exceptions
 from mitmproxy import options
+from mitmproxy import proxy
 from mitmproxy.builtins import script
 from mitmproxy.flow import master
 
@@ -57,7 +58,7 @@ def test_load_script():
 class TestScript(mastertest.MasterTest):
     def test_simple(self):
         o = options.Options()
-        m = master.FlowMaster(o, None)
+        m = master.FlowMaster(o, proxy.DummyServer())
         sc = script.Script(
             tutils.test_data.path(
                 "data/addonscripts/recorder.py"
@@ -78,7 +79,7 @@ class TestScript(mastertest.MasterTest):
 
     def test_reload(self):
         o = options.Options()
-        m = mastertest.RecordingMaster(o, None)
+        m = mastertest.RecordingMaster(o, proxy.DummyServer())
         with tutils.tmpdir():
             with open("foo.py", "w"):
                 pass
@@ -96,7 +97,7 @@ class TestScript(mastertest.MasterTest):
 
     def test_exception(self):
         o = options.Options()
-        m = mastertest.RecordingMaster(o, None)
+        m = mastertest.RecordingMaster(o, proxy.DummyServer())
         sc = script.Script(
             tutils.test_data.path("data/addonscripts/error.py")
         )
@@ -111,7 +112,7 @@ class TestScript(mastertest.MasterTest):
 
     def test_addon(self):
         o = options.Options()
-        m = master.FlowMaster(o, None)
+        m = master.FlowMaster(o, proxy.DummyServer())
         sc = script.Script(
             tutils.test_data.path(
                 "data/addonscripts/addon.py"
@@ -144,7 +145,7 @@ class TestCutTraceback:
 class TestScriptLoader(mastertest.MasterTest):
     def test_run_once(self):
         o = options.Options(scripts=[])
-        m = master.FlowMaster(o, None)
+        m = master.FlowMaster(o, proxy.DummyServer())
         sl = script.ScriptLoader()
         m.addons.add(sl)
 
@@ -168,7 +169,7 @@ class TestScriptLoader(mastertest.MasterTest):
 
     def test_simple(self):
         o = options.Options(scripts=[])
-        m = master.FlowMaster(o, None)
+        m = master.FlowMaster(o, proxy.DummyServer())
         sc = script.ScriptLoader()
         m.addons.add(sc)
         assert len(m.addons) == 1
@@ -183,7 +184,7 @@ class TestScriptLoader(mastertest.MasterTest):
 
     def test_dupes(self):
         o = options.Options(scripts=["one", "one"])
-        m = master.FlowMaster(o, None)
+        m = master.FlowMaster(o, proxy.DummyServer())
         sc = script.ScriptLoader()
         tutils.raises(exceptions.OptionsError, m.addons.add, o, sc)
 
@@ -197,7 +198,7 @@ class TestScriptLoader(mastertest.MasterTest):
                 "%s %s" % (rec, "c"),
             ]
         )
-        m = mastertest.RecordingMaster(o, None)
+        m = mastertest.RecordingMaster(o, proxy.DummyServer())
         sc = script.ScriptLoader()
         m.addons.add(sc)
 

test/mitmproxy/builtins/test_serverplayback.py

@@ -3,6 +3,7 @@ from .. import tutils, mastertest
 import netlib.tutils
 from mitmproxy.builtins import serverplayback
 from mitmproxy import options
+from mitmproxy import proxy
 from mitmproxy import exceptions
 
 
@@ -239,7 +240,7 @@ class TestServerPlayback:
     def test_server_playback_full(self):
         s = serverplayback.ServerPlayback()
         o = options.Options(refresh_server_playback = True, keepserving=False)
-        m = mastertest.RecordingMaster(o, None)
+        m = mastertest.RecordingMaster(o, proxy.DummyServer())
         m.addons.add(s)
 
         f = tutils.tflow()
@@ -268,7 +269,7 @@ class TestServerPlayback:
     def test_server_playback_kill(self):
         s = serverplayback.ServerPlayback()
         o = options.Options(refresh_server_playback = True, replay_kill_extra=True)
-        m = mastertest.RecordingMaster(o, None)
+        m = mastertest.RecordingMaster(o, proxy.DummyServer())
         m.addons.add(s)
 
         f = tutils.tflow()

test/mitmproxy/builtins/test_setheaders.py

@@ -2,12 +2,13 @@ from .. import tutils, mastertest
 
 from mitmproxy.builtins import setheaders
 from mitmproxy import options
+from mitmproxy import proxy
 
 
 class TestSetHeaders(mastertest.MasterTest):
     def mkmaster(self, **opts):
         o = options.Options(**opts)
-        m = mastertest.RecordingMaster(o, None)
+        m = mastertest.RecordingMaster(o, proxy.DummyServer())
         sh = setheaders.SetHeaders()
         m.addons.add(sh)
         return m, sh

test/mitmproxy/builtins/test_stickyauth.py

@@ -2,12 +2,13 @@ from .. import tutils, mastertest
 from mitmproxy.builtins import stickyauth
 from mitmproxy.flow import master
 from mitmproxy import options
+from mitmproxy import proxy
 
 
 class TestStickyAuth(mastertest.MasterTest):
     def test_simple(self):
         o = options.Options(stickyauth = ".*")
-        m = master.FlowMaster(o, None)
+        m = master.FlowMaster(o, proxy.DummyServer())
         sa = stickyauth.StickyAuth()
         m.addons.add(sa)
 

test/mitmproxy/builtins/test_stickycookie.py

@@ -2,6 +2,7 @@ from .. import tutils, mastertest
 from mitmproxy.builtins import stickycookie
 from mitmproxy.flow import master
 from mitmproxy import options
+from mitmproxy import proxy
 from netlib import tutils as ntutils
 
 
@@ -13,7 +14,7 @@ def test_domain_match():
 class TestStickyCookie(mastertest.MasterTest):
     def mk(self):
         o = options.Options(stickycookie = ".*")
-        m = master.FlowMaster(o, None)
+        m = master.FlowMaster(o, proxy.DummyServer())
         sc = stickycookie.StickyCookie()
         m.addons.add(sc)
         return m, sc

test/mitmproxy/builtins/test_streambodies.py

@@ -1,6 +1,7 @@
 from .. import tutils, mastertest
 from mitmproxy.flow import master
 from mitmproxy import options
+from mitmproxy import proxy
 
 from mitmproxy.builtins import streambodies
 
@@ -8,7 +9,7 @@ from mitmproxy.builtins import streambodies
 class TestStreamBodies(mastertest.MasterTest):
     def test_simple(self):
         o = options.Options(stream_large_bodies = 10)
-        m = master.FlowMaster(o, None)
+        m = master.FlowMaster(o, proxy.DummyServer())
         sa = streambodies.StreamBodies()
         m.addons.add(sa)
 

test/mitmproxy/console/test_master.py

@@ -2,6 +2,7 @@ import gc
 
 import netlib.tutils
 from mitmproxy import console
+from mitmproxy import proxy
 from mitmproxy.console import common
 
 from .. import tutils, mastertest
@@ -115,7 +116,7 @@ class TestMaster(mastertest.MasterTest):
         if "verbosity" not in options:
             options["verbosity"] = 0
         o = console.master.Options(**options)
-        return console.master.ConsoleMaster(None, o)
+        return console.master.ConsoleMaster(o, proxy.DummyServer())
 
     def test_basic(self):
         m = self.mkmaster()

test/mitmproxy/script/test_concurrent.py

@@ -2,6 +2,7 @@ from test.mitmproxy import tutils, mastertest
 from mitmproxy import controller
 from mitmproxy.builtins import script
 from mitmproxy import options
+from mitmproxy import proxy
 from mitmproxy.flow import master
 import time
 
@@ -15,7 +16,7 @@ class Thing:
 class TestConcurrent(mastertest.MasterTest):
     @tutils.skip_appveyor
     def test_concurrent(self):
-        m = master.FlowMaster(options.Options(), None)
+        m = master.FlowMaster(options.Options(), proxy.DummyServer())
         sc = script.Script(
             tutils.test_data.path(
                 "data/addonscripts/concurrent_decorator.py"
@@ -32,7 +33,7 @@ class TestConcurrent(mastertest.MasterTest):
         raise ValueError("Script never acked")
 
     def test_concurrent_err(self):
-        m = mastertest.RecordingMaster(options.Options(), None)
+        m = mastertest.RecordingMaster(options.Options(), proxy.DummyServer())
         sc = script.Script(
             tutils.test_data.path(
                 "data/addonscripts/concurrent_decorator_err.py"

test/mitmproxy/test_addons.py

@@ -1,6 +1,7 @@
 from mitmproxy import addons
 from mitmproxy import controller
 from mitmproxy import options
+from mitmproxy import proxy
 
 
 class TAddon:
@@ -13,7 +14,7 @@ class TAddon:
 
 def test_simple():
     o = options.Options()
-    m = controller.Master(o)
+    m = controller.Master(o, proxy.DummyServer(o))
     a = addons.Addons(m)
     a.add(TAddon("one"))
     assert a.get("one")

test/mitmproxy/test_controller.py

@@ -7,7 +7,7 @@ from mitmproxy import controller
 import queue
 
 from mitmproxy.exceptions import Kill, ControlException
-from mitmproxy.proxy import DummyServer
+from mitmproxy import proxy
 from netlib.tutils import raises
 
 
@@ -26,7 +26,7 @@ class TestMaster:
                 # Speed up test
                 super().tick(0)
 
-        m = DummyMaster(None)
+        m = DummyMaster(None, proxy.DummyServer(None))
         assert not m.should_exit.is_set()
         msg = TMsg()
         msg.reply = controller.DummyReply()
@@ -35,9 +35,7 @@ class TestMaster:
         assert m.should_exit.is_set()
 
     def test_server_simple(self):
-        m = controller.Master(None)
+        m = controller.Master(None, proxy.DummyServer(None))
-        s = DummyServer(None)
-        m.add_server(s)
         m.start()
         m.shutdown()
         m.start()

test/mitmproxy/test_dump.py

@@ -1,7 +1,7 @@
 import os
 import io
 
-from mitmproxy import dump, flow, exceptions
+from mitmproxy import dump, flow, exceptions, proxy
 from . import tutils, mastertest
 
 
@@ -16,7 +16,7 @@ class TestDumpMaster(mastertest.MasterTest):
         if "flow_detail" not in options:
             options["flow_detail"] = 0
         o = dump.Options(filtstr=flt, tfile=io.StringIO(), **options)
-        return dump.DumpMaster(None, o)
+        return dump.DumpMaster(o, proxy.DummyServer())
 
     def test_basic(self):
         for i in (1, 2, 3):
@@ -41,14 +41,14 @@ class TestDumpMaster(mastertest.MasterTest):
             tfile=io.StringIO(),
             flow_detail=1
         )
-        m = dump.DumpMaster(None, o)
+        m = dump.DumpMaster(o, proxy.DummyServer())
         f = tutils.tflow(err=True)
         m.error(f)
         assert "error" in o.tfile.getvalue()
 
     def test_replay(self):
         o = dump.Options(server_replay=["nonexistent"], replay_kill_extra=True)
-        tutils.raises(exceptions.OptionsError, dump.DumpMaster, None, o)
+        tutils.raises(exceptions.OptionsError, dump.DumpMaster, o, proxy.DummyServer())
 
         with tutils.tmpdir() as t:
             p = os.path.join(t, "rep")
@@ -57,7 +57,7 @@ class TestDumpMaster(mastertest.MasterTest):
             o = dump.Options(server_replay=[p], replay_kill_extra=True)
             o.verbosity = 0
             o.flow_detail = 0
-            m = dump.DumpMaster(None, o)
+            m = dump.DumpMaster(o, proxy.DummyServer())
 
             self.cycle(m, b"content")
             self.cycle(m, b"content")
@@ -65,13 +65,13 @@ class TestDumpMaster(mastertest.MasterTest):
             o = dump.Options(server_replay=[p], replay_kill_extra=False)
             o.verbosity = 0
             o.flow_detail = 0
-            m = dump.DumpMaster(None, o)
+            m = dump.DumpMaster(o, proxy.DummyServer())
             self.cycle(m, b"nonexistent")
 
             o = dump.Options(client_replay=[p], replay_kill_extra=False)
             o.verbosity = 0
             o.flow_detail = 0
-            m = dump.DumpMaster(None, o)
+            m = dump.DumpMaster(o, proxy.DummyServer())
 
     def test_read(self):
         with tutils.tmpdir() as t:
@@ -106,7 +106,7 @@ class TestDumpMaster(mastertest.MasterTest):
         )
         o.verbosity = 0
         o.flow_detail = 0
-        m = dump.DumpMaster(None, o)
+        m = dump.DumpMaster(o, proxy.DummyServer())
         f = self.cycle(m, b"content")
         assert f.request.content == b"foo"
 
@@ -117,7 +117,7 @@ class TestDumpMaster(mastertest.MasterTest):
         )
         o.verbosity = 0
         o.flow_detail = 0
-        m = dump.DumpMaster(None, o)
+        m = dump.DumpMaster(o, proxy.DummyServer())
         f = self.cycle(m, b"content")
         assert f.request.headers["one"] == "two"
 

test/mitmproxy/test_examples.py

@@ -4,6 +4,7 @@ import shlex
 
 from mitmproxy import options
 from mitmproxy import contentviews
+from mitmproxy import proxy
 from mitmproxy.builtins import script
 from mitmproxy.flow import master
 
@@ -31,7 +32,7 @@ class RaiseMaster(master.FlowMaster):
 def tscript(cmd, args=""):
     o = options.Options()
     cmd = example_dir.path(cmd) + " " + args
-    m = RaiseMaster(o, None)
+    m = RaiseMaster(o, proxy.DummyServer())
     sc = script.Script(cmd)
     m.addons.add(sc)
     return m, sc

test/mitmproxy/test_flow.py

@@ -363,7 +363,7 @@ class TestSerialize:
     def test_load_flows(self):
         r = self._treader()
         s = flow.State()
-        fm = flow.FlowMaster(None, None)
+        fm = flow.FlowMaster(None, DummyServer())
         fm.addons.add(s)
         fm.load_flows(r)
         assert len(s.flows) == 6
@@ -423,7 +423,7 @@ class TestSerialize:
 class TestFlowMaster:
 
     def test_replay(self):
-        fm = flow.FlowMaster(None, None)
+        fm = flow.FlowMaster(None, DummyServer())
         f = tutils.tflow(resp=True)
         f.request.content = None
         tutils.raises("missing", fm.replay_request, f)
@@ -435,12 +435,12 @@ class TestFlowMaster:
         tutils.raises("live", fm.replay_request, f)
 
     def test_create_flow(self):
-        fm = flow.FlowMaster(None, None)
+        fm = flow.FlowMaster(None, DummyServer())
         assert fm.create_request("GET", "http", "example.com", 80, "/")
 
     def test_all(self):
         s = flow.State()
-        fm = flow.FlowMaster(None, None)
+        fm = flow.FlowMaster(None, DummyServer())
         fm.addons.add(s)
         f = tutils.tflow(req=None)
         fm.clientconnect(f.client_conn)

test/mitmproxy/test_flow_state.py

@@ -1,11 +1,12 @@
 from mitmproxy import flow
+from mitmproxy import proxy
 from . import tutils
 
 
 class TestState:
     def test_duplicate_flow(self):
         s = flow.State()
-        fm = flow.FlowMaster(None, None)
+        fm = flow.FlowMaster(None, proxy.DummyServer())
         fm.addons.add(s)
         f = tutils.tflow(resp=True)
         fm.load_flow(f)

test/mitmproxy/test_web_app.py

@@ -1,12 +1,13 @@
 import tornado.testing
 
+from mitmproxy import proxy
 from mitmproxy.web import app, master
 
 
 class TestApp(tornado.testing.AsyncHTTPTestCase):
     def get_app(self):
         o = master.Options()
-        m = master.WebMaster(None, o)
+        m = master.WebMaster(o, proxy.DummyServer())
         return app.Application(m, None, None)
 
     def test_index(self):

test/mitmproxy/test_web_master.py

@@ -1,11 +1,12 @@
 from mitmproxy.web import master
+from mitmproxy import proxy
 from . import mastertest
 
 
 class TestWebMaster(mastertest.MasterTest):
     def mkmaster(self, **options):
         o = master.Options(**options)
-        return master.WebMaster(None, o)
+        return master.WebMaster(o, proxy.DummyServer(o))
 
     def test_basic(self):
         m = self.mkmaster()