Merge pull request #4069 from mplattner/maplocal-addon

MapLocal addon

docs/src/content/overview-features.md

@@ -11,6 +11,7 @@ menu:
 
 - [Anticache](#anticache)
 - [Client-side replay](#client-side-replay)
+- [Map Local](#map-local)
 - [Map Remote](#map-remote)
 - [Modify Body](#modify-body)
 - [Modify Headers](#modify-headers)
@@ -43,6 +44,81 @@ You may want to use client-side replay in conjunction with the `anticache`
 option, to make sure the server responds with complete data.
 
 
+## Map Local
+
+The `map_local` option lets you specify an arbitrary number of patterns that
+define redirections of HTTP requests to local files or directories.
+The local file is fetched instead of the original resource
+and transparently returned to the client.
+
+`map_local` patterns look like this:
+
+```
+|url-regex|local-path
+|flow-filter|url-regex|local-path
+```
+
+* **local-path** is the file or directory that should be served to the client.
+
+* **url-regex** is a regular expression applied on the request URL. It must match for a redirect to take place.
+
+* **flow-filter** is an optional mitmproxy [filter expression]({{< relref "concepts-filters">}})
+that additionally constrains which requests will be redirected.
+
+### Examples
+
+Pattern | Description
+------- | -----------
+`\|example.com/main.js\|~/main-local.js` | Replace `example.com/main.js` with `~/main-local.js`.
+`\|example.com/static\|~/static` | Replace `example.com/static/foo/bar.css` with `~/static/foo/bar.css`.
+`\|example.com/static/foo\|~/static` | Replace `example.com/static/foo/bar.css` with `~/static/bar.css`.
+`\|~m GET\|example.com/static\|~/static` | Replace `example.com/static/foo/bar.css` with `~/static/foo/bar.css` (but only for GET requests).
+
+### Details
+
+If *local-path* is a file, this file will always be served. File changes will be reflected immediately, there is no caching.
+
+If *local-path* is a directory, *url-regex* is used to split the request URL in two parts and part on the right is appended to *local-path*, excluding the query string.
+However, if *url-regex* contains a regex capturing group, this behavior changes and the first capturing group is appended instead (and query strings are not stripped). 
+Special characters are mapped to `_`. If the file cannot be found, `/index.html` is appended and we try again. Directory traversal outside of the originally specified directory is not possible.
+
+To illustrate this, consider the following example which maps all requests for `example.org/css*` to the local directory `~/static-css`.
+
+<pre>
+                  ┌── url regex ──┬─ local path ─┐
+map_local option: |<span style="color:#f92672">example.com/css</span>|<span style="color:#82b719">~/static-css</span>
+                   <!--                     -->         │
+                   <!--                     -->         │    URL is split here
+                   <!--                     -->         ▼            ▼
+HTTP Request URL: https://<span style="color:#f92672">example.com/css</span><span style="color:#66d9ef">/print/main.css</span><span style="color:#bbb">?timestamp=123</span>
+                          <!--                     -->               <!--                            -->      │        <!--                         -->        ▼
+                          <!--                     -->               <!--                            -->      ▼        <!--                         -->      query string is ignored
+Served File:      Preferred: <span style="color:#82b719">~/static-css</span><span style="color:#66d9ef">/print/main.css</span>
+                   Fallback: <span style="color:#82b719">~/static-css</span><span style="color:#66d9ef">/print/main.css</span>/index.html
+                  Otherwise: 404 response without content
+</pre>
+
+If the file depends on the query string, we can use regex capturing groups. In this example, all `GET` requests for 
+`example.org/index.php?page=<page-name>` are mapped to `~/static-dir/<page-name>`:
+
+<pre>
+                    flow
+                  ┌filter┬─────────── url regex ───────────┬─ local path ─┐
+map_local option: |~m GET|<span style="color:#f92672">example.com/index.php\\?page=</span><span style="color:#66d9ef">(.+)</span>|<span style="color:#82b719">~/static-dir</span>
+                          <!--                     -->  │                          <!--                            --> │
+                          <!--                     -->  │                          <!--                            --> │ regex group = suffix
+                          <!--                     -->  ▼                          <!--                            --> ▼
+HTTP Request URL: https://<span style="color:#f92672">example.com/index.php?page=</span><span style="color:#66d9ef">aboutus</span></span>
+                          <!--                     -->                           <!--                            -->   │
+                          <!--                     -->                           <!--                            -->   ▼
+Served File:                 Preferred: <span style="color:#82b719">~/static-dir</span>/<span style="color:#66d9ef">aboutus</span>
+                              Fallback: <span style="color:#82b719">~/static-dir</span>/<span style="color:#66d9ef">aboutus</span>/index.html
+                             Otherwise: 404 response without content
+</pre>
+
+
+
+
 ## Map Remote
 
 The `map_remote` option lets you specify an arbitrary number of patterns that
@@ -51,22 +127,19 @@ The substituted URL is fetched instead of the original resource
 and the corresponding HTTP response is returned transparently to the client.
 Note that if the original destination uses HTTP2, the substituted destination
 needs to support HTTP2 as well, otherwise the substituted request may fail.
-`map_remote` patterns looks like this:
+`map_remote` patterns look like this:
 
 ```
-|flow-filter|regex|replacement
-|flow-filter|regex|@file-path
-|regex|replacement
-|regex|@file-path
+|flow-filter|url-regex|replacement
+|url-regex|replacement
 ```
 
 * **flow-filter** is an optional mitmproxy [filter expression]({{< relref "concepts-filters">}})
-that defines which requests a replacement applies to.
+that defines which requests the `map_remote` option applies to.
 
-* **regex** is a valid Python regular expression that defines what gets replaced in the URLs of requests.
+* **url-regex** is a valid Python regular expression that defines what gets replaced in the URLs of requests.
 
-* **replacement** is a string literal that is substituted in. If the replacement string
-literal starts with `@` as in `@file-path`, it is treated as a **file path** from which the replacement is read.
+* **replacement** is a string literal that is substituted in.
 
 The _separator_ is arbitrary, and is defined by the first character.
 
@@ -93,16 +166,16 @@ The `modify_body` option lets you specify an arbitrary number of patterns that
 define replacements within bodies of flows. `modify_body` patterns look like this:
 
 ```
-/flow-filter/regex/replacement
-/flow-filter/regex/@file-path
-/regex/replacement
-/regex/@file-path
+/flow-filter/body-regex/replacement
+/flow-filter/body-regex/@file-path
+/body-regex/replacement
+/body-regex/@file-path
 ```
 
 * **flow-filter** is an optional mitmproxy [filter expression]({{< relref "concepts-filters">}})
 that defines which flows a replacement applies to.
 
-* **regex** is a valid Python regular expression that defines what gets replaced.
+* **body-regex** is a valid Python regular expression that defines what gets replaced.
 
 * **replacement** is a string literal that is substituted in. If the replacement string
 literal starts with `@` as in `@file-path`, it is treated as a **file path** from which the replacement is read.

mitmproxy/addons/__init__.py

@@ -14,6 +14,7 @@ from mitmproxy.addons import proxyauth
 from mitmproxy.addons import script
 from mitmproxy.addons import serverplayback
 from mitmproxy.addons import mapremote
+from mitmproxy.addons import maplocal
 from mitmproxy.addons import modifybody
 from mitmproxy.addons import modifyheaders
 from mitmproxy.addons import stickyauth
@@ -41,6 +42,7 @@ def default_addons():
         script.ScriptLoader(),
         serverplayback.ServerPlayback(),
         mapremote.MapRemote(),
+        maplocal.MapLocal(),
         modifybody.ModifyBody(),
         modifyheaders.ModifyHeaders(),
         stickyauth.StickyAuth(),

mitmproxy/addons/maplocal.py

@@ -0,0 +1,151 @@
+import mimetypes
+import re
+import typing
+import urllib.parse
+from pathlib import Path
+
+from werkzeug.security import safe_join
+
+from mitmproxy import ctx, exceptions, flowfilter, http, version
+from mitmproxy.utils.spec import parse_spec
+
+
+class MapLocalSpec(typing.NamedTuple):
+    matches: flowfilter.TFilter
+    regex: str
+    local_path: Path
+
+
+def parse_map_local_spec(option: str) -> MapLocalSpec:
+    filter, regex, replacement = parse_spec(option)
+
+    try:
+        re.compile(regex)
+    except re.error as e:
+        raise ValueError(f"Invalid regular expression {regex!r} ({e})")
+
+    try:
+        path = Path(replacement).expanduser().resolve(strict=True)
+    except FileNotFoundError as e:
+        raise ValueError(f"Invalid file path: {replacement} ({e})")
+
+    return MapLocalSpec(filter, regex, path)
+
+
+def _safe_path_join(root: Path, untrusted: str) -> Path:
+    """Join a Path element with an untrusted str.
+
+    This is a convenience wrapper for werkzeug's safe_join,
+    raising a ValueError if the path is malformed."""
+    untrusted_parts = Path(untrusted).parts
+    joined = safe_join(
+        root.as_posix(),
+        *untrusted_parts
+    )
+    if joined is None:
+        raise ValueError("Untrusted paths.")
+    return Path(joined)
+
+
+def file_candidates(url: str, spec: MapLocalSpec) -> typing.List[Path]:
+    """
+    Get all potential file candidates given a URL and a mapping spec ordered by preference.
+    This function already assumes that the spec regex matches the URL.
+    """
+    m = re.search(spec.regex, url)
+    assert m
+    if m.groups():
+        suffix = m.group(1)
+    else:
+        suffix = re.split(spec.regex, url, maxsplit=1)[1]
+        suffix = suffix.split("?")[0]  # remove query string
+        suffix = suffix.strip("/")
+
+    if suffix:
+        decoded_suffix = urllib.parse.unquote(suffix)
+        suffix_candidates = [decoded_suffix, f"{decoded_suffix}/index.html"]
+
+        escaped_suffix = re.sub(r"[^0-9a-zA-Z\-_.=(),/]", "_", decoded_suffix)
+        if decoded_suffix != escaped_suffix:
+            suffix_candidates.extend([escaped_suffix, f"{escaped_suffix}/index.html"])
+        try:
+            return [
+                _safe_path_join(spec.local_path, x)
+                for x in suffix_candidates
+            ]
+        except ValueError:
+            return []
+    else:
+        return [spec.local_path / "index.html"]
+
+
+class MapLocal:
+    def __init__(self):
+        self.replacements: typing.List[MapLocalSpec] = []
+
+    def load(self, loader):
+        loader.add_option(
+            "map_local", typing.Sequence[str], [],
+            """
+            Map remote resources to a local file using a pattern of the form
+            "[/flow-filter]/url-regex/file-or-directory-path", where the
+            separator can be any character.
+            """
+        )
+
+    def configure(self, updated):
+        if "map_local" in updated:
+            self.replacements = []
+            for option in ctx.options.map_local:
+                try:
+                    spec = parse_map_local_spec(option)
+                except ValueError as e:
+                    raise exceptions.OptionsError(f"Cannot parse map_local option {option}: {e}") from e
+
+                self.replacements.append(spec)
+
+    def request(self, flow: http.HTTPFlow) -> None:
+        if flow.reply and flow.reply.has_message:
+            return
+
+        url = flow.request.pretty_url
+
+        all_candidates = []
+        for spec in self.replacements:
+            if spec.matches(flow) and re.search(spec.regex, url):
+                if spec.local_path.is_file():
+                    candidates = [spec.local_path]
+                else:
+                    candidates = file_candidates(url, spec)
+                all_candidates.extend(candidates)
+
+                local_file = None
+                for candidate in candidates:
+                    if candidate.is_file():
+                        local_file = candidate
+                        break
+
+                if local_file:
+                    headers = {
+                        "Server": version.MITMPROXY
+                    }
+                    mimetype = mimetypes.guess_type(str(local_file))[0]
+                    if mimetype:
+                        headers["Content-Type"] = mimetype
+
+                    try:
+                        contents = local_file.read_bytes()
+                    except IOError as e:
+                        ctx.log.warn(f"Could not read file: {e}")
+                        continue
+
+                    flow.response = http.HTTPResponse.make(
+                        200,
+                        contents,
+                        headers
+                    )
+                    # only set flow.response once, for the first matching rule
+                    return
+        if all_candidates:
+            flow.response = http.HTTPResponse.make(404)
+            ctx.log.info(f"None of the local file candidates exist: {', '.join(str(x) for x in all_candidates)}")

mitmproxy/addons/mapremote.py

@@ -1,22 +1,38 @@
 import re
 import typing
 
-from mitmproxy import exceptions, http
-from mitmproxy import ctx
-from mitmproxy.addons.modifyheaders import parse_modify_spec, ModifySpec
+from mitmproxy import ctx, exceptions, flowfilter, http
+from mitmproxy.utils.spec import parse_spec
+
+
+class MapRemoteSpec(typing.NamedTuple):
+    matches: flowfilter.TFilter
+    subject: str
+    replacement: str
+
+
+def parse_map_remote_spec(option: str) -> MapRemoteSpec:
+    spec = MapRemoteSpec(*parse_spec(option))
+
+    try:
+        re.compile(spec.subject)
+    except re.error as e:
+        raise ValueError(f"Invalid regular expression {spec.subject!r} ({e})")
+
+    return spec
 
 
 class MapRemote:
     def __init__(self):
-        self.replacements: typing.List[ModifySpec] = []
+        self.replacements: typing.List[MapRemoteSpec] = []
 
     def load(self, loader):
         loader.add_option(
             "map_remote", typing.Sequence[str], [],
             """
-            Replacement pattern of the form "[/flow-filter]/regex/[@]replacement", where
-            the separator can be any character. The @ allows to provide a file path that
-            is used to read the replacement string.
+            Map remote resources to another remote URL using a pattern of the form
+            "[/flow-filter]/url-regex/replacement", where the separator can
+            be any character.
             """
         )
 
@@ -25,7 +41,7 @@ class MapRemote:
             self.replacements = []
             for option in ctx.options.map_remote:
                 try:
-                    spec = parse_modify_spec(option, True)
+                    spec = parse_map_remote_spec(option)
                 except ValueError as e:
                     raise exceptions.OptionsError(f"Cannot parse map_remote option {option}: {e}") from e
 
@@ -36,14 +52,8 @@ class MapRemote:
             return
         for spec in self.replacements:
             if spec.matches(flow):
-                try:
-                    replacement = spec.read_replacement()
-                except IOError as e:
-                    ctx.log.warn(f"Could not read replacement file: {e}")
-                    continue
-
-                url = flow.request.pretty_url.encode("utf8", "surrogateescape")
-                new_url = re.sub(spec.subject, replacement, url)
+                url = flow.request.pretty_url
+                new_url = re.sub(spec.subject, spec.replacement, url)
                 # this is a bit messy: setting .url also updates the host header,
                 # so we really only do that if the replacement affected the URL.
                 if url != new_url:

mitmproxy/addons/modifybody.py

@@ -1,8 +1,7 @@
 import re
 import typing
 
-from mitmproxy import exceptions
-from mitmproxy import ctx
+from mitmproxy import ctx, exceptions
 from mitmproxy.addons.modifyheaders import parse_modify_spec, ModifySpec
 
 

mitmproxy/addons/modifyheaders.py

@@ -2,11 +2,10 @@ import re
 import typing
 from pathlib import Path
 
-from mitmproxy import exceptions, http
-from mitmproxy import flowfilter
+from mitmproxy import ctx, exceptions, flowfilter, http
 from mitmproxy.net.http import Headers
 from mitmproxy.utils import strutils
-from mitmproxy import ctx
+from mitmproxy.utils.spec import parse_spec
 
 
 class ModifySpec(typing.NamedTuple):
@@ -29,49 +28,10 @@ class ModifySpec(typing.NamedTuple):
             return strutils.escaped_str_to_bytes(self.replacement_str)
 
 
-def _match_all(flow) -> bool:
-    return True
+def parse_modify_spec(option: str, subject_is_regex: bool) -> ModifySpec:
+    flow_filter, subject_str, replacement = parse_spec(option)
 
-
-def parse_modify_spec(option, subject_is_regex: bool) -> ModifySpec:
-    """
-        The form for the modify_* options is as follows:
-
-            * modify_headers: [/flow-filter]/header-name/[@]header-value
-            * modify_body: [/flow-filter]/search-regex/[@]replace
-
-        The @ allows to provide a file path that is used to read the respective option.
-        Both ModifyHeaders and ModifyBody use ModifySpec to represent a single rule.
-
-        The first character specifies the separator. Example:
-
-            :~q:foo:bar
-
-        If only two clauses are specified, the flow filter is set to
-        match universally (i.e. ".*"). Example:
-
-            /foo/bar
-
-        Clauses are parsed from left to right. Extra separators are taken to be
-        part of the final clause. For instance, the last parameter (header-value or
-        replace) below is "foo/bar/":
-
-            /one/two/foo/bar/
-    """
-    sep, rem = option[0], option[1:]
-    parts = rem.split(sep, 2)
-    if len(parts) == 2:
-        flow_filter = _match_all
-        subject, replacement = parts
-    elif len(parts) == 3:
-        flow_filter_pattern, subject, replacement = parts
-        flow_filter = flowfilter.parse(flow_filter_pattern)  # type: ignore
-        if not flow_filter:
-            raise ValueError(f"Invalid filter pattern: {flow_filter_pattern}")
-    else:
-        raise ValueError("Invalid number of parameters (2 or 3 are expected)")
-
-    subject = strutils.escaped_str_to_bytes(subject)
+    subject = strutils.escaped_str_to_bytes(subject_str)
     if subject_is_regex:
         try:
             re.compile(subject)

mitmproxy/tools/cmdline.py

@@ -85,6 +85,10 @@ def common_options(parser, opts):
     group = parser.add_argument_group("Map Remote")
     opts.make_parser(group, "map_remote", metavar="PATTERN", short="M")
 
+    # Map Local
+    group = parser.add_argument_group("Map Local")
+    opts.make_parser(group, "map_local", metavar="PATTERN")
+
     # Modify Body
     group = parser.add_argument_group("Modify Body")
     opts.make_parser(group, "modify_body", metavar="PATTERN", short="B")

mitmproxy/utils/spec.py

@@ -0,0 +1,28 @@
+import typing
+from mitmproxy import flowfilter
+
+
+def _match_all(flow) -> bool:
+    return True
+
+
+def parse_spec(option: str) -> typing.Tuple[flowfilter.TFilter, str, str]:
+    """
+    Parse strings in the following format:
+
+        [/flow-filter]/subject/replacement
+
+    """
+    sep, rem = option[0], option[1:]
+    parts = rem.split(sep, 2)
+    if len(parts) == 2:
+        subject, replacement = parts
+        return _match_all, subject, replacement
+    elif len(parts) == 3:
+        patt, subject, replacement = parts
+        flow_filter = flowfilter.parse(patt)
+        if not flow_filter:
+            raise ValueError(f"Invalid filter pattern: {patt}")
+        return flow_filter, subject, replacement
+    else:
+        raise ValueError("Invalid number of parameters (2 or 3 are expected)")

test/mitmproxy/addons/test_maplocal.py

@@ -0,0 +1,186 @@
+import sys
+from pathlib import Path
+
+import pytest
+
+from mitmproxy.addons.maplocal import MapLocal, MapLocalSpec, file_candidates
+from mitmproxy.utils.spec import parse_spec
+from mitmproxy.test import taddons
+from mitmproxy.test import tflow
+
+
+@pytest.mark.parametrize(
+    "url,spec,expected_candidates",
+    [
+        # trailing slashes
+        ("https://example.com/foo", ":example.com/foo:/tmp", ["/tmp/index.html"]),
+        ("https://example.com/foo/", ":example.com/foo:/tmp", ["/tmp/index.html"]),
+        ("https://example.com/foo", ":example.com/foo:/tmp/", ["/tmp/index.html"]),
+    ] + [
+        # simple prefixes
+        ("http://example.com/foo/bar.jpg", ":example.com/foo:/tmp", ["/tmp/bar.jpg", "/tmp/bar.jpg/index.html"]),
+        ("https://example.com/foo/bar.jpg", ":example.com/foo:/tmp", ["/tmp/bar.jpg", "/tmp/bar.jpg/index.html"]),
+        ("https://example.com/foo/bar.jpg?query", ":example.com/foo:/tmp", ["/tmp/bar.jpg", "/tmp/bar.jpg/index.html"]),
+        ("https://example.com/foo/bar/baz.jpg", ":example.com/foo:/tmp",
+         ["/tmp/bar/baz.jpg", "/tmp/bar/baz.jpg/index.html"]),
+        ("https://example.com/foo/bar.jpg", ":/foo/bar.jpg:/tmp", ["/tmp/index.html"]),
+    ] + [
+        # URL decode and special characters
+        ("http://example.com/foo%20bar.jpg", ":example.com:/tmp", [
+            "/tmp/foo bar.jpg",
+            "/tmp/foo bar.jpg/index.html",
+            "/tmp/foo_bar.jpg",
+            "/tmp/foo_bar.jpg/index.html"
+        ]),
+        ("http://example.com/fóobår.jpg", ":example.com:/tmp", [
+            "/tmp/fóobår.jpg",
+            "/tmp/fóobår.jpg/index.html",
+            "/tmp/f_ob_r.jpg",
+            "/tmp/f_ob_r.jpg/index.html"
+        ]),
+    ] + [
+        # index.html
+        ("https://example.com/foo", ":example.com/foo:/tmp", ["/tmp/index.html"]),
+        ("https://example.com/foo/", ":example.com/foo:/tmp", ["/tmp/index.html"]),
+        ("https://example.com/foo/bar", ":example.com/foo:/tmp", ["/tmp/bar", "/tmp/bar/index.html"]),
+        ("https://example.com/foo/bar/", ":example.com/foo:/tmp", ["/tmp/bar", "/tmp/bar/index.html"]),
+    ] + [
+        # regex
+        (
+                "https://example/view.php?f=foo.jpg",
+                ":example/view.php\\?f=(.+):/tmp",
+                ["/tmp/foo.jpg", "/tmp/foo.jpg/index.html"]
+        ), (
+                "https://example/results?id=1&foo=2",
+                ":example/(results\\?id=.+):/tmp",
+                [
+                    "/tmp/results?id=1&foo=2",
+                    "/tmp/results?id=1&foo=2/index.html",
+                    "/tmp/results_id=1_foo=2",
+                    "/tmp/results_id=1_foo=2/index.html"
+                ]
+        ),
+    ] + [
+        # test directory traversal detection
+        ("https://example.com/../../../../../../etc/passwd", ":example.com:/tmp", []),
+        # this is slightly hacky, but werkzeug's behavior differs per system.
+        ("https://example.com/C:\\foo.txt", ":example.com:/tmp", [] if sys.platform == "win32" else [
+            "/tmp/C:\\foo.txt",
+            "/tmp/C:\\foo.txt/index.html",
+            "/tmp/C__foo.txt",
+            "/tmp/C__foo.txt/index.html"
+        ]),
+        ("https://example.com//etc/passwd", ":example.com:/tmp", ["/tmp/etc/passwd", "/tmp/etc/passwd/index.html"]),
+    ]
+)
+def test_file_candidates(url, spec, expected_candidates):
+    # we circumvent the path existence checks here to simplify testing
+    filt, subj, repl = parse_spec(spec)
+    spec = MapLocalSpec(filt, subj, Path(repl))
+
+    candidates = file_candidates(url, spec)
+    assert [x.as_posix() for x in candidates] == expected_candidates
+
+
+class TestMapLocal:
+
+    def test_configure(self, tmpdir):
+        ml = MapLocal()
+        with taddons.context(ml) as tctx:
+            tctx.configure(ml, map_local=["/foo/bar/" + str(tmpdir)])
+            with pytest.raises(Exception, match="Invalid regular expression"):
+                tctx.configure(ml, map_local=["/foo/+/" + str(tmpdir)])
+            with pytest.raises(Exception, match="Invalid file path"):
+                tctx.configure(ml, map_local=["/foo/.+/three"])
+
+    def test_simple(self, tmpdir):
+        ml = MapLocal()
+
+        with taddons.context(ml) as tctx:
+            tmpfile = tmpdir.join("foo.jpg")
+            tmpfile.write("foo")
+            tctx.configure(
+                ml,
+                map_local=[
+                    "|//example.org/images|" + str(tmpdir)
+                ]
+            )
+            f = tflow.tflow()
+            f.request.url = b"https://example.org/images/foo.jpg"
+            ml.request(f)
+            assert f.response.content == b"foo"
+
+            tmpfile = tmpdir.join("images", "bar.jpg")
+            tmpfile.write("bar", ensure=True)
+            tctx.configure(
+                ml,
+                map_local=[
+                    "|//example.org|" + str(tmpdir)
+                ]
+            )
+            f = tflow.tflow()
+            f.request.url = b"https://example.org/images/bar.jpg"
+            ml.request(f)
+            assert f.response.content == b"bar"
+
+            tmpfile = tmpdir.join("foofoobar.jpg")
+            tmpfile.write("foofoobar", ensure=True)
+            tctx.configure(
+                ml,
+                map_local=[
+                    "|example.org/foo/foo/bar.jpg|" + str(tmpfile)
+                ]
+            )
+            f = tflow.tflow()
+            f.request.url = b"https://example.org/foo/foo/bar.jpg"
+            ml.request(f)
+            assert f.response.content == b"foofoobar"
+
+    @pytest.mark.asyncio
+    async def test_nonexistent_files(self, tmpdir, monkeypatch):
+        ml = MapLocal()
+
+        with taddons.context(ml) as tctx:
+            tctx.configure(
+                ml,
+                map_local=[
+                    "|example.org/css|" + str(tmpdir)
+                ]
+            )
+            f = tflow.tflow()
+            f.request.url = b"https://example.org/css/nonexistent"
+            ml.request(f)
+            assert f.response.status_code == 404
+            assert await tctx.master.await_log("None of the local file candidates exist")
+
+            tmpfile = tmpdir.join("foo.jpg")
+            tmpfile.write("foo")
+            tctx.configure(
+                ml,
+                map_local=[
+                    "|//example.org/images|" + str(tmpfile)
+                ]
+            )
+            tmpfile.remove()
+            monkeypatch.setattr(Path, "is_file", lambda x: True)
+            f = tflow.tflow()
+            f.request.url = b"https://example.org/images/foo.jpg"
+            ml.request(f)
+            assert await tctx.master.await_log("could not read file")
+
+    def test_has_reply(self, tmpdir):
+        ml = MapLocal()
+        with taddons.context(ml) as tctx:
+            tmpfile = tmpdir.join("foo.jpg")
+            tmpfile.write("foo")
+            tctx.configure(
+                ml,
+                map_local=[
+                    "|//example.org/images|" + str(tmpfile)
+                ]
+            )
+            f = tflow.tflow()
+            f.request.url = b"https://example.org/images/foo.jpg"
+            f.kill()
+            ml.request(f)
+            assert not f.response

test/mitmproxy/addons/test_mapremote.py

@@ -11,13 +11,8 @@ class TestMapRemote:
         mr = mapremote.MapRemote()
         with taddons.context(mr) as tctx:
             tctx.configure(mr, map_remote=["one/two/three"])
-            with pytest.raises(Exception, match="Cannot parse map_remote .* Invalid number"):
-                tctx.configure(mr, map_remote=["/"])
-            with pytest.raises(Exception, match="Cannot parse map_remote .* Invalid filter"):
-                tctx.configure(mr, map_remote=["/~b/two/three"])
-            with pytest.raises(Exception, match="Cannot parse map_remote .* Invalid regular expression"):
+            with pytest.raises(Exception, match="Invalid regular expression"):
                 tctx.configure(mr, map_remote=["/foo/+/three"])
-            tctx.configure(mr, map_remote=["/a/b/c/"])
 
     def test_simple(self):
         mr = mapremote.MapRemote()
@@ -42,41 +37,3 @@ class TestMapRemote:
             f.kill()
             mr.request(f)
             assert f.request.url == "https://example.org/images/test.jpg"
-
-
-class TestMapRemoteFile:
-    def test_simple(self, tmpdir):
-        mr = mapremote.MapRemote()
-        with taddons.context(mr) as tctx:
-            tmpfile = tmpdir.join("replacement")
-            tmpfile.write("mitmproxy.org")
-            tctx.configure(
-                mr,
-                map_remote=["|example.org|@" + str(tmpfile)]
-            )
-            f = tflow.tflow()
-            f.request.url = b"https://example.org/test"
-            mr.request(f)
-            assert f.request.url == "https://mitmproxy.org/test"
-
-    @pytest.mark.asyncio
-    async def test_nonexistent(self, tmpdir):
-        mr = mapremote.MapRemote()
-        with taddons.context(mr) as tctx:
-            with pytest.raises(Exception, match="Invalid file path"):
-                tctx.configure(
-                    mr,
-                    map_remote=[":~q:example.org:@nonexistent"]
-                )
-
-            tmpfile = tmpdir.join("replacement")
-            tmpfile.write("mitmproxy.org")
-            tctx.configure(
-                mr,
-                map_remote=["|example.org|@" + str(tmpfile)]
-            )
-            tmpfile.remove()
-            f = tflow.tflow()
-            f.request.url = b"https://example.org/test"
-            mr.request(f)
-            assert await tctx.master.await_log("could not read")

test/mitmproxy/addons/test_modifybody.py

@@ -12,7 +12,6 @@ class TestModifyBody:
             tctx.configure(mb, modify_body=["one/two/three"])
             with pytest.raises(Exception, match="Cannot parse modify_body"):
                 tctx.configure(mb, modify_body=["/"])
-            tctx.configure(mb, modify_body=["/a/b/c/"])
 
     def test_simple(self):
         mb = modifybody.ModifyBody()

test/mitmproxy/addons/test_modifyheaders.py

@@ -22,15 +22,6 @@ def test_parse_modify_spec():
     assert spec.subject == b"bar"
     assert spec.read_replacement() == b"voing"
 
-    with pytest.raises(ValueError, match="Invalid number of parameters"):
-        parse_modify_spec("/", False)
-
-    with pytest.raises(ValueError, match="Invalid filter pattern"):
-        parse_modify_spec("/~b/one/two", False)
-
-    with pytest.raises(ValueError, match="Invalid filter pattern"):
-        parse_modify_spec("/~b/one/two", False)
-
     with pytest.raises(ValueError, match="Invalid regular expression"):
         parse_modify_spec("/[/two", True)
 

test/mitmproxy/utils/test_spec.py

@@ -0,0 +1,20 @@
+import pytest
+from mitmproxy.utils.spec import parse_spec
+
+
+def test_parse_spec():
+    flow_filter, subject, replacement = parse_spec("/foo/bar/voing")
+    assert flow_filter.pattern == "foo"
+    assert subject == "bar"
+    assert replacement == "voing"
+
+    flow_filter, subject, replacement = parse_spec("/bar/voing")
+    assert flow_filter(1) is True
+    assert subject == "bar"
+    assert replacement == "voing"
+
+    with pytest.raises(ValueError, match="Invalid number of parameters"):
+        parse_spec("/")
+
+    with pytest.raises(ValueError, match="Invalid filter pattern"):
+        parse_spec("/~b/one/two")