Use existing urlencoding conventions when re-encoding edited form #1946 (#2022)

mitmproxy/net/http/request.py

@@ -373,7 +373,7 @@ class Request(message.Message):
         This will overwrite the existing content if there is one.
         """
         self.headers["content-type"] = "application/x-www-form-urlencoded"
-        self.content = mitmproxy.net.http.url.encode(form_data).encode()
+        self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode()
 
     @urlencoded_form.setter
     def urlencoded_form(self, value):

mitmproxy/net/http/url.py

@@ -82,11 +82,24 @@ def unparse(scheme, host, port, path=""):
     return "%s://%s%s" % (scheme, hostport(scheme, host, port), path)
 
 
-def encode(s: Sequence[Tuple[str, str]]) -> str:
+def encode(s: Sequence[Tuple[str, str]], similar_to: str=None) -> str:
     """
         Takes a list of (key, value) tuples and returns a urlencoded string.
+        If similar_to is passed, the output is formatted similar to the provided urlencoded string.
     """
-    return urllib.parse.urlencode(s, False, errors="surrogateescape")
+
+    remove_trailing_equal = False
+    if similar_to:
+        remove_trailing_equal = any("=" not in param for param in similar_to.split("&"))
+
+    encoded = urllib.parse.urlencode(s, False, errors="surrogateescape")
+
+    if remove_trailing_equal:
+        encoded = encoded.replace("=&", "&")
+        if encoded[-1] == '=':
+            encoded = encoded[:-1]
+
+    return encoded
 
 
 def decode(s):

test/mitmproxy/net/http/test_url.py

@@ -85,6 +85,26 @@ surrogates_quoted = (
 )
 
 
+def test_empty_key_trailing_equal_sign():
+    """
+    Some HTTP clients don't send trailing equal signs for parameters without assigned value, e.g. they send
+        foo=bar&baz&qux=quux
+    instead of
+        foo=bar&baz=&qux=quux
+    The respective behavior of encode() should be driven by a reference string given in similar_to parameter
+    """
+    reference_without_equal = "key1=val1&key2&key3=val3"
+    reference_with_equal = "key1=val1&key2=&key3=val3"
+
+    post_data_empty_key_middle = [('one', 'two'), ('emptykey', ''), ('three', 'four')]
+    post_data_empty_key_end = [('one', 'two'), ('three', 'four'), ('emptykey', '')]
+
+    assert url.encode(post_data_empty_key_middle, similar_to = reference_with_equal) == "one=two&emptykey=&three=four"
+    assert url.encode(post_data_empty_key_end, similar_to = reference_with_equal) == "one=two&three=four&emptykey="
+    assert url.encode(post_data_empty_key_middle, similar_to = reference_without_equal) == "one=two&emptykey&three=four"
+    assert url.encode(post_data_empty_key_end, similar_to = reference_without_equal) == "one=two&three=four&emptykey"
+
+
 def test_encode():
     assert url.encode([('foo', 'bar')])
     assert url.encode([('foo', surrogates)])