Remove watchdog, solidify script testing

- Remove the watchdog dependency. We now just stat the script file every 2
seconds to check for an updated mtime.
- Further solidify our script testing, and in particular make the example tests
nicer. These should exemplify how we want users to test their own addon
scripts. More work on addon testing to follow.

mitmproxy/addonmanager.py

@@ -129,9 +129,13 @@ class AddonManager:
 
     def register(self, addon):
         """
-            Register an addon and all its sub-addons with the manager without
+            Register an addon, call its load event, and then register all its
-            adding it to the chain. This should be used by addons that
+            sub-addons. This should be used by addons that dynamically manage
-            dynamically manage addons. Must be called within a current context.
+            addons.
+
+            If the calling addon is already running, it should follow with
+            running and configure events. Must be called within a current
+            context.
         """
         for a in traverse([addon]):
             name = _get_name(a)

mitmproxy/addons/script.py

@@ -1,15 +1,12 @@
 import os
 import importlib
-import threading
+import time
 import sys
 
 from mitmproxy import addonmanager
 from mitmproxy import exceptions
 from mitmproxy import ctx
 
-import watchdog.events
-from watchdog.observers import polling
-
 
 def load_script(actx, path):
     if not os.path.exists(path):
@@ -28,60 +25,30 @@ def load_script(actx, path):
         sys.path[:] = oldpath
 
 
-class ReloadHandler(watchdog.events.FileSystemEventHandler):
-    def __init__(self, callback):
-        self.callback = callback
-
-    def filter(self, event):
-        """
-            Returns True only when .py file is changed
-        """
-        if event.is_directory:
-            return False
-        if os.path.basename(event.src_path).startswith("."):
-            return False
-        if event.src_path.endswith(".py"):
-            return True
-        return False
-
-    def on_modified(self, event):
-        if self.filter(event):
-            self.callback()
-
-    def on_created(self, event):
-        if self.filter(event):
-            self.callback()
-
-
 class Script:
     """
         An addon that manages a single script.
     """
+    ReloadInterval = 2
+
     def __init__(self, path):
         self.name = "scriptmanager:" + path
         self.path = path
         self.ns = None
-        self.observer = None
 
         self.last_options = None
-        self.should_reload = threading.Event()
+        self.last_load = 0
-
+        self.last_mtime = 0
-    def load(self, l):
-        self.ns = load_script(ctx, self.path)
 
     @property
     def addons(self):
-        if self.ns is not None:
+        return [self.ns] if self.ns else []
-            return [self.ns]
-        return []
-
-    def reload(self):
-        self.should_reload.set()
 
     def tick(self):
-        if self.should_reload.is_set():
+        if time.time() - self.last_load > self.ReloadInterval:
-            self.should_reload.clear()
+            mtime = os.stat(self.path).st_mtime
-            ctx.log.info("Reloading script: %s" % self.name)
+            if mtime > self.last_mtime:
+                ctx.log.info("Loading script: %s" % self.name)
                 if self.ns:
                     ctx.master.addons.remove(self.ns)
                 self.ns = load_script(ctx, self.path)
@@ -89,18 +56,18 @@ class Script:
                     # We're already running, so we have to explicitly register and
                     # configure the addon
                     ctx.master.addons.register(self.ns)
-                self.configure(self.last_options, self.last_options.keys())
+                    ctx.master.addons.invoke_addon(self.ns, "running")
+                    ctx.master.addons.invoke_addon(
+                        self.ns,
+                        "configure",
+                        self.last_options,
+                        self.last_options.keys()
+                    )
+                self.last_load = time.time()
+                self.last_mtime = mtime
 
     def configure(self, options, updated):
         self.last_options = options
-        if not self.observer:
-            self.observer = polling.PollingObserver()
-            # Bind the handler to the real underlying master object
-            self.observer.schedule(
-                ReloadHandler(self.reload),
-                os.path.dirname(self.path) or "."
-            )
-            self.observer.start()
 
 
 class ScriptLoader:

mitmproxy/test/taddons.py

@@ -112,9 +112,12 @@ class context:
             )
 
     def script(self, path):
+        """
+            Loads a script from path, and returns the enclosed addon.
+        """
         sc = script.Script(path)
         loader = addonmanager.Loader(self.master)
-        sc.load(loader)
+        self.master.addons.invoke_addon(sc, "load", loader)
-        for a in addonmanager.traverse(sc.addons):
+        self.configure(sc)
-            getattr(a, "load", lambda x: None)(loader)
+        self.master.addons.invoke_addon(sc, "tick")
-        return sc
+        return sc.addons[0] if sc.addons else None

setup.py

@@ -80,7 +80,6 @@ setup(
         "ruamel.yaml>=0.13.2, <0.15",
         "tornado>=4.3, <4.6",
         "urwid>=1.3.1, <1.4",
-        "watchdog>=0.8.3, <0.9",
         "brotlipy>=0.5.1, <0.7",
         "sortedcontainers>=1.5.4, <1.6",
         # transitive from cryptography, we just blacklist here.

test/examples/test_examples.py

@@ -1,9 +1,4 @@
-from mitmproxy import options
 from mitmproxy import contentviews
-from mitmproxy import proxy
-from mitmproxy import master
-from mitmproxy.addons import script
-
 from mitmproxy.test import tflow
 from mitmproxy.test import tutils
 from mitmproxy.test import taddons
@@ -14,34 +9,17 @@ from ..mitmproxy import tservers
 example_dir = tutils.test_data.push("../examples")
 
 
-class ScriptError(Exception):
-    pass
-
-
-class RaiseMaster(master.Master):
-    def add_log(self, e, level):
-        if level in ("warn", "error"):
-            raise ScriptError(e)
-
-
-def tscript(cmd, args=""):
-    o = options.Options()
-    cmd = example_dir.path(cmd)
-    m = RaiseMaster(o, proxy.DummyServer())
-    sc = script.Script(cmd)
-    m.addons.add(sc)
-    return m, sc
-
-
 class TestScripts(tservers.MasterTest):
     def test_add_header(self):
-        m, _ = tscript("simple/add_header.py")
+        with taddons.context() as tctx:
+            a = tctx.script(example_dir.path("simple/add_header.py"))
             f = tflow.tflow(resp=tutils.tresp())
-        m.addons.handle_lifecycle("response", f)
+            a.response(f)
             assert f.response.headers["newheader"] == "foo"
 
     def test_custom_contentviews(self):
-        m, sc = tscript("simple/custom_contentview.py")
+        with taddons.context() as tctx:
+            tctx.script(example_dir.path("simple/custom_contentview.py"))
             swapcase = contentviews.get("swapcase")
             _, fmt = swapcase(b"<html>Test!</html>")
             assert any(b'tEST!' in val[0][1] for val in fmt)
@@ -61,55 +39,61 @@ class TestScripts(tservers.MasterTest):
             assert b'iframe' in content and b'evil_iframe' in content
 
     def test_modify_form(self):
-        m, sc = tscript("simple/modify_form.py")
+        with taddons.context() as tctx:
+            sc = tctx.script(example_dir.path("simple/modify_form.py"))
 
             form_header = Headers(content_type="application/x-www-form-urlencoded")
             f = tflow.tflow(req=tutils.treq(headers=form_header))
-        m.addons.handle_lifecycle("request", f)
+            sc.request(f)
 
             assert f.request.urlencoded_form["mitmproxy"] == "rocks"
 
             f.request.headers["content-type"] = ""
-        m.addons.handle_lifecycle("request", f)
+            sc.request(f)
             assert list(f.request.urlencoded_form.items()) == [("foo", "bar")]
 
     def test_modify_querystring(self):
-        m, sc = tscript("simple/modify_querystring.py")
+        with taddons.context() as tctx:
+            sc = tctx.script(example_dir.path("simple/modify_querystring.py"))
             f = tflow.tflow(req=tutils.treq(path="/search?q=term"))
 
-        m.addons.handle_lifecycle("request", f)
+            sc.request(f)
             assert f.request.query["mitmproxy"] == "rocks"
 
             f.request.path = "/"
-        m.addons.handle_lifecycle("request", f)
+            sc.request(f)
             assert f.request.query["mitmproxy"] == "rocks"
 
     def test_redirect_requests(self):
-        m, sc = tscript("simple/redirect_requests.py")
+        with taddons.context() as tctx:
+            sc = tctx.script(example_dir.path("simple/redirect_requests.py"))
             f = tflow.tflow(req=tutils.treq(host="example.org"))
-        m.addons.handle_lifecycle("request", f)
+            sc.request(f)
             assert f.request.host == "mitmproxy.org"
 
     def test_send_reply_from_proxy(self):
-        m, sc = tscript("simple/send_reply_from_proxy.py")
+        with taddons.context() as tctx:
+            sc = tctx.script(example_dir.path("simple/send_reply_from_proxy.py"))
             f = tflow.tflow(req=tutils.treq(host="example.com", port=80))
-        m.addons.handle_lifecycle("request", f)
+            sc.request(f)
             assert f.response.content == b"Hello World"
 
     def test_dns_spoofing(self):
-        m, sc = tscript("complex/dns_spoofing.py")
+        with taddons.context() as tctx:
+            sc = tctx.script(example_dir.path("complex/dns_spoofing.py"))
+
             original_host = "example.com"
 
             host_header = Headers(host=original_host)
             f = tflow.tflow(req=tutils.treq(headers=host_header, port=80))
 
-        m.addons.handle_lifecycle("requestheaders", f)
+            tctx.master.addons.invoke_addon(sc, "requestheaders", f)
 
             # Rewrite by reverse proxy mode
             f.request.scheme = "https"
             f.request.port = 443
 
-        m.addons.handle_lifecycle("request", f)
+            tctx.master.addons.invoke_addon(sc, "request", f)
 
             assert f.request.scheme == "http"
             assert f.request.port == 80

test/mitmproxy/addons/test_script.py

@@ -1,7 +1,6 @@
 import traceback
 import sys
 import time
-import watchdog.events
 import pytest
 
 from unittest import mock
@@ -16,34 +15,6 @@ from mitmproxy import master
 from mitmproxy.addons import script
 
 
-class Called:
-    def __init__(self):
-        self.called = False
-
-    def __call__(self, *args, **kwargs):
-        self.called = True
-
-
-def test_reloadhandler():
-    rh = script.ReloadHandler(Called())
-    assert not rh.filter(watchdog.events.DirCreatedEvent("path"))
-    assert not rh.filter(watchdog.events.FileModifiedEvent("/foo/.bar"))
-    assert not rh.filter(watchdog.events.FileModifiedEvent("/foo/bar"))
-    assert rh.filter(watchdog.events.FileModifiedEvent("/foo/bar.py"))
-
-    assert not rh.callback.called
-    rh.on_modified(watchdog.events.FileModifiedEvent("/foo/bar"))
-    assert not rh.callback.called
-    rh.on_modified(watchdog.events.FileModifiedEvent("/foo/bar.py"))
-    assert rh.callback.called
-    rh.callback.called = False
-
-    rh.on_created(watchdog.events.FileCreatedEvent("foo"))
-    assert not rh.callback.called
-    rh.on_created(watchdog.events.FileCreatedEvent("foo.py"))
-    assert rh.callback.called
-
-
 def test_load_script():
     with taddons.context() as tctx:
         ns = script.load_script(
@@ -89,6 +60,8 @@ class TestScript:
                 )
             )
             tctx.master.addons.add(sc)
+            tctx.configure(sc)
+            sc.tick()
 
             rec = tctx.master.addons.get("recorder")
 
@@ -107,10 +80,12 @@ class TestScript:
             f.write("\n")
             sc = script.Script(str(f))
             tctx.configure(sc)
-            for _ in range(5):
+            sc.tick()
-                sc.reload()
+            for _ in range(3):
+                sc.last_load, sc.last_mtime = 0, 0
                 sc.tick()
                 time.sleep(0.1)
+            tctx.master.has_log("Loading")
 
     def test_exception(self):
         with taddons.context() as tctx:
@@ -118,10 +93,12 @@ class TestScript:
                 tutils.test_data.path("mitmproxy/data/addonscripts/error.py")
             )
             tctx.master.addons.add(sc)
+            tctx.configure(sc)
+            sc.tick()
+
             f = tflow.tflow(resp=True)
             tctx.master.addons.trigger("request", f)
 
-            assert tctx.master.logs[0].level == "error"
             tctx.master.has_log("ValueError: Error!")
             tctx.master.has_log("error.py")
 
@@ -133,8 +110,10 @@ class TestScript:
                 )
             )
             tctx.master.addons.add(sc)
+            tctx.configure(sc)
+            sc.tick()
             assert sc.ns.event_log == [
-                'scriptload', 'addonload'
+                'scriptload', 'addonload', 'scriptconfigure', 'addonconfigure'
             ]
 
 
@@ -207,21 +186,23 @@ class TestScriptLoader:
                     "%s/c.py" % rec,
                 ]
             )
-
+            tctx.master.addons.invoke_addon(sc, "tick")
             debug = [i.msg for i in tctx.master.logs if i.level == "debug"]
             assert debug == [
                 'a load',
                 'a running',
+                'a configure',
+                'a tick',
 
                 'b load',
                 'b running',
+                'b configure',
+                'b tick',
 
                 'c load',
                 'c running',
-
-                'a configure',
-                'b configure',
                 'c configure',
+                'c tick',
             ]
 
             tctx.master.logs = []
@@ -233,6 +214,7 @@ class TestScriptLoader:
                     "%s/b.py" % rec,
                 ]
             )
+
             debug = [i.msg for i in tctx.master.logs if i.level == "debug"]
             assert debug == [
                 'c configure',
@@ -248,13 +230,16 @@ class TestScriptLoader:
                     "%s/a.py" % rec,
                 ]
             )
+            tctx.master.addons.invoke_addon(sc, "tick")
 
             debug = [i.msg for i in tctx.master.logs if i.level == "debug"]
             assert debug == [
                 'c done',
                 'b done',
+                'a configure',
                 'e load',
                 'e running',
                 'e configure',
-                'a configure',
+                'e tick',
+                'a tick',
             ]

test/mitmproxy/data/addonscripts/addon.py

@@ -14,7 +14,7 @@ class Addon:
 
 
 def configure(options, updated):
-    event_log.append("addonconfigure")
+    event_log.append("scriptconfigure")
 
 
 def load(l):

test/mitmproxy/data/addonscripts/concurrent_decorator_err.py

@@ -2,5 +2,5 @@ from mitmproxy.script import concurrent
 
 
 @concurrent
-def start(opts):
+def load(v):
     pass

test/mitmproxy/script/test_concurrent.py

@@ -2,10 +2,7 @@ from mitmproxy.test import tflow
 from mitmproxy.test import tutils
 from mitmproxy.test import taddons
 
-from mitmproxy import addonmanager
 from mitmproxy import controller
-from mitmproxy.addons import script
-
 import time
 
 from .. import tservers
@@ -36,25 +33,20 @@ class TestConcurrent(tservers.MasterTest):
 
     def test_concurrent_err(self):
         with taddons.context() as tctx:
-            sc = script.Script(
+            tctx.script(
                 tutils.test_data.path(
                     "mitmproxy/data/addonscripts/concurrent_decorator_err.py"
                 )
             )
-            l = addonmanager.Loader(tctx.master)
-            sc.load(l)
             assert tctx.master.has_log("decorator not supported")
 
     def test_concurrent_class(self):
             with taddons.context() as tctx:
-                sc = script.Script(
+                sc = tctx.script(
                     tutils.test_data.path(
                         "mitmproxy/data/addonscripts/concurrent_decorator_class.py"
                     )
                 )
-                l = addonmanager.Loader(tctx.master)
-                sc.load(l)
-
                 f1, f2 = tflow.tflow(), tflow.tflow()
                 tctx.cycle(sc, f1)
                 tctx.cycle(sc, f2)