From c43a2ef8dc4aed6789aaf9116f6b18bd0e5f292e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 8 Oct 2021 17:43:52 +0200 Subject: [PATCH] improve flowfilter api: raise on invalid input, add `~all` --- examples/addons/filter-flows.py | 3 +- mitmproxy/addons/blocklist.py | 2 -- mitmproxy/addons/dumper.py | 9 +++-- mitmproxy/addons/intercept.py | 7 ++-- mitmproxy/addons/readfile.py | 13 ++++--- mitmproxy/addons/save.py | 9 +++-- mitmproxy/addons/stickyauth.py | 10 +++--- mitmproxy/addons/stickycookie.py | 10 +++--- mitmproxy/addons/view.py | 30 +++++++--------- mitmproxy/flowfilter.py | 41 ++++++++++++++++------ mitmproxy/utils/spec.py | 6 +--- test/mitmproxy/addons/test_readfile.py | 2 +- test/mitmproxy/addons/test_stickycookie.py | 2 +- test/mitmproxy/addons/test_view.py | 4 +-- test/mitmproxy/test_flowfilter.py | 6 ++-- test/mitmproxy/utils/test_spec.py | 2 +- web/src/js/ducks/_options_gen.ts | 2 -- 17 files changed, 82 insertions(+), 76 deletions(-) diff --git a/examples/addons/filter-flows.py b/examples/addons/filter-flows.py index a4e09ba3f..c5e7e6275 100644 --- a/examples/addons/filter-flows.py +++ b/examples/addons/filter-flows.py @@ -10,7 +10,8 @@ class Filter: self.filter: flowfilter.TFilter = None def configure(self, updated): - self.filter = flowfilter.parse(ctx.options.flowfilter) + if "flowfilter" in updated: + self.filter = flowfilter.parse(ctx.options.flowfilter) def load(self, l): l.add_option( diff --git a/mitmproxy/addons/blocklist.py b/mitmproxy/addons/blocklist.py index ceb6de91d..5a422cbbb 100644 --- a/mitmproxy/addons/blocklist.py +++ b/mitmproxy/addons/blocklist.py @@ -28,8 +28,6 @@ def parse_spec(option: str) -> BlockSpec: except ValueError: raise ValueError(f"Invalid HTTP status code: {status}") flow_filter = flowfilter.parse(flow_patt) - if not flow_filter: - raise ValueError(f"Invalid filter pattern: {flow_patt}") if not RESPONSES.get(status_code): raise ValueError(f"Invalid HTTP status code: {status}") diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index da0842b88..233bf8596 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -59,11 +59,10 @@ class Dumper: def configure(self, updated): if "dumper_filter" in updated: if ctx.options.dumper_filter: - self.filter = flowfilter.parse(ctx.options.dumper_filter) - if not self.filter: - raise exceptions.OptionsError( - "Invalid filter expression: %s" % ctx.options.dumper_filter - ) + try: + self.filter = flowfilter.parse(ctx.options.dumper_filter) + except ValueError as e: + raise exceptions.OptionsError(str(e)) from e else: self.filter = None diff --git a/mitmproxy/addons/intercept.py b/mitmproxy/addons/intercept.py index 5eebd6dc0..36f269bd4 100644 --- a/mitmproxy/addons/intercept.py +++ b/mitmproxy/addons/intercept.py @@ -21,9 +21,10 @@ class Intercept: def configure(self, updated): if "intercept" in updated: if ctx.options.intercept: - self.filt = flowfilter.parse(ctx.options.intercept) - if not self.filt: - raise exceptions.OptionsError(f"Invalid interception filter: {ctx.options.intercept}") + try: + self.filt = flowfilter.parse(ctx.options.intercept) + except ValueError as e: + raise exceptions.OptionsError(str(e)) from e ctx.options.intercept_active = True else: self.filt = None diff --git a/mitmproxy/addons/readfile.py b/mitmproxy/addons/readfile.py index 607b41b40..c75bc8f7d 100644 --- a/mitmproxy/addons/readfile.py +++ b/mitmproxy/addons/readfile.py @@ -30,14 +30,13 @@ class ReadFile: def configure(self, updated): if "readfile_filter" in updated: - filt = None if ctx.options.readfile_filter: - filt = flowfilter.parse(ctx.options.readfile_filter) - if not filt: - raise exceptions.OptionsError( - "Invalid readfile filter: %s" % ctx.options.readfile_filter - ) - self.filter = filt + try: + self.filter = flowfilter.parse(ctx.options.readfile_filter) + except ValueError as e: + raise exceptions.OptionsError(str(e)) from e + else: + self.filter = None async def load_flows(self, fo: typing.IO[bytes]) -> int: cnt = 0 diff --git a/mitmproxy/addons/save.py b/mitmproxy/addons/save.py index 9297e7399..3b3766dc6 100644 --- a/mitmproxy/addons/save.py +++ b/mitmproxy/addons/save.py @@ -48,11 +48,10 @@ class Save: # We're already streaming - stop the previous stream and restart if "save_stream_filter" in updated: if ctx.options.save_stream_filter: - self.filt = flowfilter.parse(ctx.options.save_stream_filter) - if not self.filt: - raise exceptions.OptionsError( - "Invalid filter specification: %s" % ctx.options.save_stream_filter - ) + try: + self.filt = flowfilter.parse(ctx.options.save_stream_filter) + except ValueError as e: + raise exceptions.OptionsError(str(e)) from e else: self.filt = None if "save_stream_file" in updated or "save_stream_filter" in updated: diff --git a/mitmproxy/addons/stickyauth.py b/mitmproxy/addons/stickyauth.py index ab0599eef..43098b2d7 100644 --- a/mitmproxy/addons/stickyauth.py +++ b/mitmproxy/addons/stickyauth.py @@ -19,12 +19,10 @@ class StickyAuth: def configure(self, updated): if "stickyauth" in updated: if ctx.options.stickyauth: - flt = flowfilter.parse(ctx.options.stickyauth) - if not flt: - raise exceptions.OptionsError( - "stickyauth: invalid filter expression: %s" % ctx.options.stickyauth - ) - self.flt = flt + try: + self.flt = flowfilter.parse(ctx.options.stickyauth) + except ValueError as e: + raise exceptions.OptionsError(str(e)) from e else: self.flt = None diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index 1651c1f66..3b67bb82c 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -43,12 +43,10 @@ class StickyCookie: def configure(self, updated): if "stickycookie" in updated: if ctx.options.stickycookie: - flt = flowfilter.parse(ctx.options.stickycookie) - if not flt: - raise exceptions.OptionsError( - "stickycookie: invalid filter expression: %s" % ctx.options.stickycookie - ) - self.flt = flt + try: + self.flt = flowfilter.parse(ctx.options.stickycookie) + except ValueError as e: + raise exceptions.OptionsError(str(e)) from e else: self.flt = None diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 13c5baf17..b3b6e4f5a 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -114,9 +114,6 @@ class OrderKeySize(_OrderKey): else: raise NotImplementedError() - -matchall = flowfilter.parse("~http | ~tcp") - orders = [ ("t", "time"), ("m", "method"), @@ -129,7 +126,7 @@ class View(collections.abc.Sequence): def __init__(self): super().__init__() self._store = collections.OrderedDict() - self.filter = matchall + self.filter = flowfilter.match_all # Should we show only marked flows? self.show_marked = False @@ -326,11 +323,10 @@ class View(collections.abc.Sequence): """ filt = None if filter_expr: - filt = flowfilter.parse(filter_expr) - if not filt: - raise exceptions.CommandError( - "Invalid interception filter: %s" % filter_expr - ) + try: + filt = flowfilter.parse(filter_expr) + except ValueError as e: + raise exceptions.CommandError(str(e)) from e self.set_filter(filt) def set_filter(self, flt: typing.Optional[flowfilter.TFilter]): @@ -454,9 +450,10 @@ class View(collections.abc.Sequence): ids = flow_spec[1:].split(",") return [i for i in self._store.values() if i.id in ids] else: - filt = flowfilter.parse(flow_spec) - if not filt: - raise exceptions.CommandError("Invalid flow filter: %s" % flow_spec) + try: + filt = flowfilter.parse(flow_spec) + except ValueError as e: + raise exceptions.CommandError(str(e)) from e return [i for i in self._store.values() if filt(i)] @command.command("view.flows.create") @@ -547,11 +544,10 @@ class View(collections.abc.Sequence): if "view_filter" in updated: filt = None if ctx.options.view_filter: - filt = flowfilter.parse(ctx.options.view_filter) - if not filt: - raise exceptions.OptionsError( - "Invalid interception filter: %s" % ctx.options.view_filter - ) + try: + filt = flowfilter.parse(ctx.options.view_filter) + except ValueError as e: + raise exceptions.OptionsError(str(e)) from e self.set_filter(filt) if "view_order" in updated: if ctx.options.view_order not in self.orders: diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index fea8e9e42..6cb5691da 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -35,7 +35,7 @@ import functools import re import sys -from typing import Callable, ClassVar, Optional, Sequence, Type +from typing import Callable, ClassVar, Optional, Sequence, Type, Protocol, Union import pyparsing as pp from mitmproxy import flow, http, tcp @@ -135,6 +135,14 @@ class FResp(_Action): return bool(f.response) +class FAll(_Action): + code = "all" + help = "Match all flows" + + def __call__(self, f: flow.Flow): + return True + + class _Rex(_Action): flags = 0 is_binary = True @@ -504,6 +512,7 @@ filter_unary: Sequence[Type[_Action]] = [ FResp, FTCP, FWebSocket, + FAll, ] filter_rex: Sequence[Type[_Rex]] = [ FBod, @@ -583,21 +592,31 @@ def _make(): bnf = _make() -TFilter = Callable[[flow.Flow], bool] -def parse(s: str) -> Optional[TFilter]: +class TFilter(Protocol): + pattern: str + + def __call__(self, f: flow.Flow) -> bool: + ... + + +def parse(s: str) -> TFilter: + """ + Parse a filter expression and return the compiled filter function. + If the filter syntax is invalid, `ValueError` is raised. + """ + if not s: + raise ValueError("Empty filter expression") try: flt = bnf.parseString(s, parseAll=True)[0] flt.pattern = s return flt - except pp.ParseException: - return None - except ValueError: - return None + except (pp.ParseException, ValueError) as e: + raise ValueError(f"Invalid filter expression: {s!r}") from e -def match(flt, flow): +def match(flt: Union[str, TFilter], flow: flow.Flow) -> bool: """ Matches a flow against a compiled filter expression. Returns True if matched, False if not. @@ -607,13 +626,15 @@ def match(flt, flow): """ if isinstance(flt, str): flt = parse(flt) - if not flt: - raise ValueError("Invalid filter expression.") if flt: return flt(flow) return True +match_all: TFilter = parse("~all") +"""A filter function that matches all flows""" + + help = [] for a in filter_unary: help.append( diff --git a/mitmproxy/utils/spec.py b/mitmproxy/utils/spec.py index c38fe92e8..309216a03 100644 --- a/mitmproxy/utils/spec.py +++ b/mitmproxy/utils/spec.py @@ -2,10 +2,6 @@ import typing from mitmproxy import flowfilter -def _match_all(flow) -> bool: - return True - - def parse_spec(option: str) -> typing.Tuple[flowfilter.TFilter, str, str]: """ Parse strings in the following format: @@ -17,7 +13,7 @@ def parse_spec(option: str) -> typing.Tuple[flowfilter.TFilter, str, str]: parts = rem.split(sep, 2) if len(parts) == 2: subject, replacement = parts - return _match_all, subject, replacement + return flowfilter.match_all, subject, replacement elif len(parts) == 3: patt, subject, replacement = parts flow_filter = flowfilter.parse(patt) diff --git a/test/mitmproxy/addons/test_readfile.py b/test/mitmproxy/addons/test_readfile.py index 8a9c7b8e1..3d2419bb5 100644 --- a/test/mitmproxy/addons/test_readfile.py +++ b/test/mitmproxy/addons/test_readfile.py @@ -43,7 +43,7 @@ class TestReadFile: rf = readfile.ReadFile() with taddons.context(rf) as tctx: tctx.configure(rf, readfile_filter="~q") - with pytest.raises(Exception, match="Invalid readfile filter"): + with pytest.raises(Exception, match="Invalid filter expression"): tctx.configure(rf, readfile_filter="~~") @pytest.mark.asyncio diff --git a/test/mitmproxy/addons/test_stickycookie.py b/test/mitmproxy/addons/test_stickycookie.py index 4493e9cc2..25fc085ab 100644 --- a/test/mitmproxy/addons/test_stickycookie.py +++ b/test/mitmproxy/addons/test_stickycookie.py @@ -16,7 +16,7 @@ class TestStickyCookie: def test_config(self): sc = stickycookie.StickyCookie() with taddons.context(sc) as tctx: - with pytest.raises(Exception, match="invalid filter"): + with pytest.raises(Exception, match="Invalid filter expression"): tctx.configure(sc, stickycookie="~b") tctx.configure(sc, stickycookie="foo") diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 61ddfdd30..0032a7823 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -268,7 +268,7 @@ def test_resolve(): assert m(tctx.command(v.resolve, "@unmarked")) == ["PUT", "GET", "PUT"] assert m(tctx.command(v.resolve, "@all")) == ["GET", "PUT", "GET", "PUT"] - with pytest.raises(exceptions.CommandError, match="Invalid flow filter"): + with pytest.raises(exceptions.CommandError, match="Invalid filter expression"): tctx.command(v.resolve, "~") @@ -608,7 +608,7 @@ def test_configure(): v = view.View() with taddons.context(v) as tctx: tctx.configure(v, view_filter="~q") - with pytest.raises(Exception, match="Invalid interception filter"): + with pytest.raises(Exception, match="Invalid filter expression"): tctx.configure(v, view_filter="~~") tctx.configure(v, view_order="method") diff --git a/test/mitmproxy/test_flowfilter.py b/test/mitmproxy/test_flowfilter.py index 9105b4243..9aa582cb3 100644 --- a/test/mitmproxy/test_flowfilter.py +++ b/test/mitmproxy/test_flowfilter.py @@ -13,10 +13,12 @@ class TestParsing: assert c.getvalue() def test_parse_err(self): - assert flowfilter.parse("~h [") is None + with pytest.raises(ValueError): + flowfilter.parse("~b") + with pytest.raises(ValueError): + flowfilter.parse("~h [") def test_simple(self): - assert not flowfilter.parse("~b") assert flowfilter.parse("~q") assert flowfilter.parse("~c 10") assert flowfilter.parse("~m foobar") diff --git a/test/mitmproxy/utils/test_spec.py b/test/mitmproxy/utils/test_spec.py index 63a063563..6cefcacc7 100644 --- a/test/mitmproxy/utils/test_spec.py +++ b/test/mitmproxy/utils/test_spec.py @@ -16,5 +16,5 @@ def test_parse_spec(): with pytest.raises(ValueError, match="Invalid number of parameters"): parse_spec("/") - with pytest.raises(ValueError, match="Invalid filter pattern"): + with pytest.raises(ValueError, match="Invalid filter expression"): parse_spec("/~b/one/two") diff --git a/web/src/js/ducks/_options_gen.ts b/web/src/js/ducks/_options_gen.ts index f077cafc7..a8bf9fce7 100644 --- a/web/src/js/ducks/_options_gen.ts +++ b/web/src/js/ducks/_options_gen.ts @@ -14,7 +14,6 @@ export interface OptionsState { ciphers_server: string | undefined client_certs: string | undefined client_replay: string[] - client_replay_concurrency: number command_history: boolean confdir: string connection_strategy: string @@ -100,7 +99,6 @@ export const defaultState: OptionsState = { ciphers_server: undefined, client_certs: undefined, client_replay: [], - client_replay_concurrency: 1, command_history: true, confdir: "~/.mitmproxy", connection_strategy: "eager",