improve flowfilter api: raise on invalid input, add ~all

examples/addons/filter-flows.py

@@ -10,7 +10,8 @@ class Filter:
         self.filter: flowfilter.TFilter = None
 
     def configure(self, updated):
-        self.filter = flowfilter.parse(ctx.options.flowfilter)
+        if "flowfilter" in updated:
+            self.filter = flowfilter.parse(ctx.options.flowfilter)
 
     def load(self, l):
         l.add_option(

mitmproxy/addons/blocklist.py

@@ -28,8 +28,6 @@ def parse_spec(option: str) -> BlockSpec:
     except ValueError:
         raise ValueError(f"Invalid HTTP status code: {status}")
     flow_filter = flowfilter.parse(flow_patt)
-    if not flow_filter:
-        raise ValueError(f"Invalid filter pattern: {flow_patt}")
     if not RESPONSES.get(status_code):
         raise ValueError(f"Invalid HTTP status code: {status}")
 

mitmproxy/addons/dumper.py

@@ -59,11 +59,10 @@ class Dumper:
     def configure(self, updated):
         if "dumper_filter" in updated:
             if ctx.options.dumper_filter:
-                self.filter = flowfilter.parse(ctx.options.dumper_filter)
-                if not self.filter:
-                    raise exceptions.OptionsError(
-                        "Invalid filter expression: %s" % ctx.options.dumper_filter
-                    )
+                try:
+                    self.filter = flowfilter.parse(ctx.options.dumper_filter)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
             else:
                 self.filter = None
 

mitmproxy/addons/intercept.py

@@ -21,9 +21,10 @@ class Intercept:
     def configure(self, updated):
         if "intercept" in updated:
             if ctx.options.intercept:
-                self.filt = flowfilter.parse(ctx.options.intercept)
-                if not self.filt:
-                    raise exceptions.OptionsError(f"Invalid interception filter: {ctx.options.intercept}")
+                try:
+                    self.filt = flowfilter.parse(ctx.options.intercept)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
                 ctx.options.intercept_active = True
             else:
                 self.filt = None

mitmproxy/addons/readfile.py

@@ -30,14 +30,13 @@ class ReadFile:
 
     def configure(self, updated):
         if "readfile_filter" in updated:
-            filt = None
             if ctx.options.readfile_filter:
-                filt = flowfilter.parse(ctx.options.readfile_filter)
-                if not filt:
-                    raise exceptions.OptionsError(
-                        "Invalid readfile filter: %s" % ctx.options.readfile_filter
-                    )
-            self.filter = filt
+                try:
+                    self.filter = flowfilter.parse(ctx.options.readfile_filter)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
+            else:
+                self.filter = None
 
     async def load_flows(self, fo: typing.IO[bytes]) -> int:
         cnt = 0

mitmproxy/addons/save.py

@@ -48,11 +48,10 @@ class Save:
         # We're already streaming - stop the previous stream and restart
         if "save_stream_filter" in updated:
             if ctx.options.save_stream_filter:
-                self.filt = flowfilter.parse(ctx.options.save_stream_filter)
-                if not self.filt:
-                    raise exceptions.OptionsError(
-                        "Invalid filter specification: %s" % ctx.options.save_stream_filter
-                    )
+                try:
+                    self.filt = flowfilter.parse(ctx.options.save_stream_filter)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
             else:
                 self.filt = None
         if "save_stream_file" in updated or "save_stream_filter" in updated:

mitmproxy/addons/stickyauth.py

@@ -19,12 +19,10 @@ class StickyAuth:
     def configure(self, updated):
         if "stickyauth" in updated:
             if ctx.options.stickyauth:
-                flt = flowfilter.parse(ctx.options.stickyauth)
-                if not flt:
-                    raise exceptions.OptionsError(
-                        "stickyauth: invalid filter expression: %s" % ctx.options.stickyauth
-                    )
-                self.flt = flt
+                try:
+                    self.flt = flowfilter.parse(ctx.options.stickyauth)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
             else:
                 self.flt = None
 

mitmproxy/addons/stickycookie.py

@@ -43,12 +43,10 @@ class StickyCookie:
     def configure(self, updated):
         if "stickycookie" in updated:
             if ctx.options.stickycookie:
-                flt = flowfilter.parse(ctx.options.stickycookie)
-                if not flt:
-                    raise exceptions.OptionsError(
-                        "stickycookie: invalid filter expression: %s" % ctx.options.stickycookie
-                    )
-                self.flt = flt
+                try:
+                    self.flt = flowfilter.parse(ctx.options.stickycookie)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
             else:
                 self.flt = None
 

mitmproxy/addons/view.py

@@ -114,9 +114,6 @@ class OrderKeySize(_OrderKey):
         else:
             raise NotImplementedError()
 
-
-matchall = flowfilter.parse("~http | ~tcp")
-
 orders = [
     ("t", "time"),
     ("m", "method"),
@@ -129,7 +126,7 @@ class View(collections.abc.Sequence):
     def __init__(self):
         super().__init__()
         self._store = collections.OrderedDict()
-        self.filter = matchall
+        self.filter = flowfilter.match_all
         # Should we show only marked flows?
         self.show_marked = False
 
@@ -326,11 +323,10 @@ class View(collections.abc.Sequence):
         """
         filt = None
         if filter_expr:
-            filt = flowfilter.parse(filter_expr)
-            if not filt:
-                raise exceptions.CommandError(
-                    "Invalid interception filter: %s" % filter_expr
-                )
+            try:
+                filt = flowfilter.parse(filter_expr)
+            except ValueError as e:
+                raise exceptions.CommandError(str(e)) from e
         self.set_filter(filt)
 
     def set_filter(self, flt: typing.Optional[flowfilter.TFilter]):
@@ -454,9 +450,10 @@ class View(collections.abc.Sequence):
             ids = flow_spec[1:].split(",")
             return [i for i in self._store.values() if i.id in ids]
         else:
-            filt = flowfilter.parse(flow_spec)
-            if not filt:
-                raise exceptions.CommandError("Invalid flow filter: %s" % flow_spec)
+            try:
+                filt = flowfilter.parse(flow_spec)
+            except ValueError as e:
+                raise exceptions.CommandError(str(e)) from e
             return [i for i in self._store.values() if filt(i)]
 
     @command.command("view.flows.create")
@@ -547,11 +544,10 @@ class View(collections.abc.Sequence):
         if "view_filter" in updated:
             filt = None
             if ctx.options.view_filter:
-                filt = flowfilter.parse(ctx.options.view_filter)
-                if not filt:
-                    raise exceptions.OptionsError(
-                        "Invalid interception filter: %s" % ctx.options.view_filter
-                    )
+                try:
+                    filt = flowfilter.parse(ctx.options.view_filter)
+                except ValueError as e:
+                    raise exceptions.OptionsError(str(e)) from e
             self.set_filter(filt)
         if "view_order" in updated:
             if ctx.options.view_order not in self.orders:

mitmproxy/flowfilter.py

@@ -35,7 +35,7 @@
 import functools
 import re
 import sys
-from typing import Callable, ClassVar, Optional, Sequence, Type
+from typing import Callable, ClassVar, Optional, Sequence, Type, Protocol, Union
 import pyparsing as pp
 
 from mitmproxy import flow, http, tcp
@@ -135,6 +135,14 @@ class FResp(_Action):
         return bool(f.response)
 
 
+class FAll(_Action):
+    code = "all"
+    help = "Match all flows"
+
+    def __call__(self, f: flow.Flow):
+        return True
+
+
 class _Rex(_Action):
     flags = 0
     is_binary = True
@@ -504,6 +512,7 @@ filter_unary: Sequence[Type[_Action]] = [
     FResp,
     FTCP,
     FWebSocket,
+    FAll,
 ]
 filter_rex: Sequence[Type[_Rex]] = [
     FBod,
@@ -583,21 +592,31 @@ def _make():
 
 
 bnf = _make()
-TFilter = Callable[[flow.Flow], bool]
 
 
-def parse(s: str) -> Optional[TFilter]:
+class TFilter(Protocol):
+    pattern: str
+
+    def __call__(self, f: flow.Flow) -> bool:
+        ...
+
+
+def parse(s: str) -> TFilter:
+    """
+    Parse a filter expression and return the compiled filter function.
+    If the filter syntax is invalid, `ValueError` is raised.
+    """
+    if not s:
+        raise ValueError("Empty filter expression")
     try:
         flt = bnf.parseString(s, parseAll=True)[0]
         flt.pattern = s
         return flt
-    except pp.ParseException:
-        return None
-    except ValueError:
-        return None
+    except (pp.ParseException, ValueError) as e:
+        raise ValueError(f"Invalid filter expression: {s!r}") from e
 
 
-def match(flt, flow):
+def match(flt: Union[str, TFilter], flow: flow.Flow) -> bool:
     """
         Matches a flow against a compiled filter expression.
         Returns True if matched, False if not.
@@ -607,13 +626,15 @@ def match(flt, flow):
     """
     if isinstance(flt, str):
         flt = parse(flt)
-        if not flt:
-            raise ValueError("Invalid filter expression.")
     if flt:
         return flt(flow)
     return True
 
 
+match_all: TFilter = parse("~all")
+"""A filter function that matches all flows"""
+
+
 help = []
 for a in filter_unary:
     help.append(

mitmproxy/utils/spec.py

@@ -2,10 +2,6 @@ import typing
 from mitmproxy import flowfilter
 
 
-def _match_all(flow) -> bool:
-    return True
-
-
 def parse_spec(option: str) -> typing.Tuple[flowfilter.TFilter, str, str]:
     """
     Parse strings in the following format:
@@ -17,7 +13,7 @@ def parse_spec(option: str) -> typing.Tuple[flowfilter.TFilter, str, str]:
     parts = rem.split(sep, 2)
     if len(parts) == 2:
         subject, replacement = parts
-        return _match_all, subject, replacement
+        return flowfilter.match_all, subject, replacement
     elif len(parts) == 3:
         patt, subject, replacement = parts
         flow_filter = flowfilter.parse(patt)

test/mitmproxy/addons/test_readfile.py

@@ -43,7 +43,7 @@ class TestReadFile:
         rf = readfile.ReadFile()
         with taddons.context(rf) as tctx:
             tctx.configure(rf, readfile_filter="~q")
-            with pytest.raises(Exception, match="Invalid readfile filter"):
+            with pytest.raises(Exception, match="Invalid filter expression"):
                 tctx.configure(rf, readfile_filter="~~")
 
     @pytest.mark.asyncio

test/mitmproxy/addons/test_stickycookie.py

@@ -16,7 +16,7 @@ class TestStickyCookie:
     def test_config(self):
         sc = stickycookie.StickyCookie()
         with taddons.context(sc) as tctx:
-            with pytest.raises(Exception, match="invalid filter"):
+            with pytest.raises(Exception, match="Invalid filter expression"):
                 tctx.configure(sc, stickycookie="~b")
 
             tctx.configure(sc, stickycookie="foo")

test/mitmproxy/addons/test_view.py

@@ -268,7 +268,7 @@ def test_resolve():
         assert m(tctx.command(v.resolve, "@unmarked")) == ["PUT", "GET", "PUT"]
         assert m(tctx.command(v.resolve, "@all")) == ["GET", "PUT", "GET", "PUT"]
 
-        with pytest.raises(exceptions.CommandError, match="Invalid flow filter"):
+        with pytest.raises(exceptions.CommandError, match="Invalid filter expression"):
             tctx.command(v.resolve, "~")
 
 
@@ -608,7 +608,7 @@ def test_configure():
     v = view.View()
     with taddons.context(v) as tctx:
         tctx.configure(v, view_filter="~q")
-        with pytest.raises(Exception, match="Invalid interception filter"):
+        with pytest.raises(Exception, match="Invalid filter expression"):
             tctx.configure(v, view_filter="~~")
 
         tctx.configure(v, view_order="method")

test/mitmproxy/test_flowfilter.py

@@ -13,10 +13,12 @@ class TestParsing:
         assert c.getvalue()
 
     def test_parse_err(self):
-        assert flowfilter.parse("~h [") is None
+        with pytest.raises(ValueError):
+            flowfilter.parse("~b")
+        with pytest.raises(ValueError):
+            flowfilter.parse("~h [")
 
     def test_simple(self):
-        assert not flowfilter.parse("~b")
         assert flowfilter.parse("~q")
         assert flowfilter.parse("~c 10")
         assert flowfilter.parse("~m foobar")

test/mitmproxy/utils/test_spec.py

@@ -16,5 +16,5 @@ def test_parse_spec():
     with pytest.raises(ValueError, match="Invalid number of parameters"):
         parse_spec("/")
 
-    with pytest.raises(ValueError, match="Invalid filter pattern"):
+    with pytest.raises(ValueError, match="Invalid filter expression"):
         parse_spec("/~b/one/two")

web/src/js/ducks/_options_gen.ts

@@ -14,7 +14,6 @@ export interface OptionsState {
     ciphers_server: string | undefined
     client_certs: string | undefined
     client_replay: string[]
-    client_replay_concurrency: number
     command_history: boolean
     confdir: string
     connection_strategy: string
@@ -100,7 +99,6 @@ export const defaultState: OptionsState = {
     ciphers_server: undefined,
     client_certs: undefined,
     client_replay: [],
-    client_replay_concurrency: 1,
     command_history: true,
     confdir: "~/.mitmproxy",
     connection_strategy: "eager",