options: add the concept of deferred settings

We've had a perpetual sequencing problem with addon startup. Users need to be
able to specify options to addons on the command-line, before addons are
actually loaded. This is only exacerbated with the new async core, where load
order can't be relied on.

This patch introduces deferred options. Options passed with "--set" on the
command line are deferred if they are unknown, and are automatically applied by
the addon manager once matching addons are registered and their options are defined.

mitmproxy/addonmanager.py

@@ -173,6 +173,7 @@ class AddonManager:
             self.lookup[name] = a
         for a in traverse([addon]):
             self.master.commands.collect_commands(a)
+        self.master.options.process_deferred()
         return addon
 
     def add(self, *addons):

mitmproxy/optmanager.py

@@ -91,9 +91,12 @@ class OptManager:
         mutation doesn't change the option state inadvertently.
     """
     def __init__(self):
-        self.__dict__["_options"] = {}
-        self.__dict__["changed"] = blinker.Signal()
-        self.__dict__["errored"] = blinker.Signal()
+        self._deferred: typing.Dict[str, str] = {}
+        self.changed = blinker.Signal()
+        self.errored = blinker.Signal()
+        # Options must be the last attribute here - after that, we raise an
+        # error for attribute assigment to unknown options.
+        self._options: typing.Dict[str, typing.Any] = {}
 
     def add_option(
         self,
@@ -168,7 +171,14 @@ class OptManager:
             raise AttributeError("No such option: %s" % attr)
 
     def __setattr__(self, attr, value):
-        self.update(**{attr: value})
+        # This is slightly tricky. We allow attributes to be set on the instance
+        # until we have an _options attribute. After that, assignment is sent to
+        # the update function, and will raise an error for unknown options.
+        opts = self.__dict__.get("_options")
+        if not opts:
+            super().__setattr__(attr, value)
+        else:
+            self.update(**{attr: value})
 
     def keys(self):
         return set(self._options.keys())
@@ -272,12 +282,44 @@ class OptManager:
             options=options
         )
 
-    def set(self, *spec):
+    def set(self, *spec, defer=False):
+        """
+            Takes a list of set specification in standard form (option=value).
+            Options that are known are updated immediately. If defer is true,
+            options that are not known are deferred, and will be set once they
+            are added.
+        """
         vals = {}
+        unknown = {}
         for i in spec:
-            vals.update(self._setspec(i))
+            parts = i.split("=", maxsplit=1)
+            if len(parts) == 1:
+                optname, optval = parts[0], None
+            else:
+                optname, optval = parts[0], parts[1]
+            if optname in self._options:
+                vals[optname] = self.parse_setval(optname, optval)
+            else:
+                unknown[optname] = optval
+        if defer:
+            self._deferred.update(unknown)
+        elif unknown:
+            raise exceptions.OptionsError("Unknown options: %s" % ", ".join(unknown.keys()))
         self.update(**vals)
 
+    def process_deferred(self):
+        """
+            Processes options that were deferred in previous calls to set, and
+            have since been added.
+        """
+        update = {}
+        for optname, optval in self._deferred.items():
+            if optname in self._options:
+                update[optname] = self.parse_setval(optname, optval)
+        self.update(**update)
+        for k in update.keys():
+            del self._deferred[k]
+
     def parse_setval(self, optname: str, optstr: typing.Optional[str]) -> typing.Any:
         """
             Convert a string to a value appropriate for the option type.
@@ -316,16 +358,6 @@ class OptManager:
                 return getattr(self, optname) + [optstr]
         raise NotImplementedError("Unsupported option type: %s", o.typespec)
 
-    def _setspec(self, spec):
-        d = {}
-        parts = spec.split("=", maxsplit=1)
-        if len(parts) == 1:
-            optname, optval = parts[0], None
-        else:
-            optname, optval = parts[0], parts[1]
-        d[optname] = self.parse_setval(optname, optval)
-        return d
-
     def make_parser(self, parser, optname, metavar=None, short=None):
         """
             Auto-Create a command-line parser entry for a named option. If the

mitmproxy/tools/main.py

@@ -110,7 +110,7 @@ def run(
         if args.commands:
             master.commands.dump()
             sys.exit(0)
-        opts.set(*args.setoptions)
+        opts.set(*args.setoptions, defer=True)
         if extra:
             opts.update(**extra(args))
 

test/mitmproxy/test_optmanager.py

@@ -426,4 +426,13 @@ def test_set():
     assert opts.seqstr == []
 
     with pytest.raises(exceptions.OptionsError):
-        opts.set("nonexistent=wobble")
+        opts.set("deferred=wobble")
+
+    opts.set("deferred=wobble", defer=True)
+    assert "deferred" in opts._deferred
+    opts.process_deferred()
+    assert "deferred" in opts._deferred
+    opts.add_option("deferred", str, "default", "help")
+    opts.process_deferred()
+    assert "deferred" not in opts._deferred
+    assert opts.deferred == "wobble"