fix #1858

examples/complex/har_dump.py

@@ -136,7 +136,7 @@ def response(flow):
 
     if flow.request.method in ["POST", "PUT", "PATCH"]:
         params = [
-            {"name": a.decode("utf8", "surrogateescape"), "value": b.decode("utf8", "surrogateescape")}
+            {"name": a, "value": b}
             for a, b in flow.request.urlencoded_form.items(multi=True)
         ]
         entry["request"]["postData"] = {

mitmproxy/addons/serverplayback.py

@@ -1,9 +1,10 @@
-import urllib
 import hashlib
+import urllib
+from typing import Any  # noqa
+from typing import List  # noqa
 
-from mitmproxy.utils import strutils
-from mitmproxy import exceptions
 from mitmproxy import ctx
+from mitmproxy import exceptions
 from mitmproxy import io
 
 
@@ -36,17 +37,20 @@ class ServerPlayback:
         _, _, path, _, query, _ = urllib.parse.urlparse(r.url)
         queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
 
-        key = [str(r.port), str(r.scheme), str(r.method), str(path)]
+        key = [str(r.port), str(r.scheme), str(r.method), str(path)]  # type: List[Any]
         if not self.options.server_replay_ignore_content:
-            form_contents = r.urlencoded_form or r.multipart_form
-            if self.options.server_replay_ignore_payload_params and form_contents:
-                params = [
-                    strutils.always_bytes(i)
-                    for i in self.options.server_replay_ignore_payload_params
-                ]
-                for p in form_contents.items(multi=True):
-                    if p[0] not in params:
-                        key.append(p)
+            if self.options.server_replay_ignore_payload_params and r.multipart_form:
+                key.extend(
+                    (k, v)
+                    for k, v in r.multipart_form.items(multi=True)
+                    if k.decode(errors="replace") not in self.options.server_replay_ignore_payload_params
+                )
+            elif self.options.server_replay_ignore_payload_params and r.urlencoded_form:
+                key.extend(
+                    (k, v)
+                    for k, v in r.urlencoded_form.items(multi=True)
+                    if k not in self.options.server_replay_ignore_payload_params
+                )
             else:
                 key.append(str(r.raw_content))
 

mitmproxy/net/http/request.py

@@ -350,6 +350,8 @@ class Request(message.Message):
         The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
         An empty multidict.MultiDictView if the content-type indicates non-form data
         or the content could not be parsed.
+
+        Starting with mitmproxy 1.0, key and value are strings.
         """
         return multidict.MultiDictView(
             self._get_urlencoded_form,
@@ -360,7 +362,7 @@ class Request(message.Message):
         is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower()
         if is_valid_content_type:
             try:
-                return tuple(mitmproxy.net.http.url.decode(self.content))
+                return tuple(mitmproxy.net.http.url.decode(self.content.decode()))
             except ValueError:
                 pass
         return ()
@@ -381,7 +383,10 @@ class Request(message.Message):
     def multipart_form(self):
         """
         The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
-        None if the content-type indicates non-form data.
+        An empty multidict.MultiDictView if the content-type indicates non-form data
+        or the content could not be parsed.
+
+        Key and value are bytes.
         """
         return multidict.MultiDictView(
             self._get_multipart_form,

test/mitmproxy/net/http/test_request.py

@@ -255,11 +255,11 @@ class TestRequestUtils:
         assert not request.urlencoded_form
 
         request.headers["Content-Type"] = "application/x-www-form-urlencoded"
-        assert list(request.urlencoded_form.items()) == [(b"foobar", b"baz")]
+        assert list(request.urlencoded_form.items()) == [("foobar", "baz")]
 
     def test_set_urlencoded_form(self):
         request = treq()
-        request.urlencoded_form = [(b'foo', b'bar'), (b'rab', b'oof')]
+        request.urlencoded_form = [('foo', 'bar'), ('rab', 'oof')]
         assert request.headers["Content-Type"] == "application/x-www-form-urlencoded"
         assert request.content
 

test/mitmproxy/test_examples.py

@@ -68,11 +68,11 @@ class TestScripts(mastertest.MasterTest):
         f = tflow.tflow(req=tutils.treq(headers=form_header))
         m.request(f)
 
-        assert f.request.urlencoded_form[b"mitmproxy"] == b"rocks"
+        assert f.request.urlencoded_form["mitmproxy"] == "rocks"
 
         f.request.headers["content-type"] = ""
         m.request(f)
-        assert list(f.request.urlencoded_form.items()) == [(b"foo", b"bar")]
+        assert list(f.request.urlencoded_form.items()) == [("foo", "bar")]
 
     def test_modify_querystring(self):
         m, sc = tscript("simple/modify_querystring.py")