From ffdbccd5719d9d5a51cbeb61b3516390d9384f78 Mon Sep 17 00:00:00 2001 From: mame82 Date: Thu, 16 Feb 2017 15:03:27 +0100 Subject: [PATCH] Use existing urlencoding conventions when re-encoding edited form #1946 (#2022) --- mitmproxy/net/http/request.py | 2 +- mitmproxy/net/http/url.py | 17 +++++++++++++++-- test/mitmproxy/net/http/test_url.py | 20 ++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 822f82298..68a11ce73 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -373,7 +373,7 @@ class Request(message.Message): This will overwrite the existing content if there is one. """ self.headers["content-type"] = "application/x-www-form-urlencoded" - self.content = mitmproxy.net.http.url.encode(form_data).encode() + self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode() @urlencoded_form.setter def urlencoded_form(self, value): diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index ff3d52641..86ce97642 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -82,11 +82,24 @@ def unparse(scheme, host, port, path=""): return "%s://%s%s" % (scheme, hostport(scheme, host, port), path) -def encode(s: Sequence[Tuple[str, str]]) -> str: +def encode(s: Sequence[Tuple[str, str]], similar_to: str=None) -> str: """ Takes a list of (key, value) tuples and returns a urlencoded string. + If similar_to is passed, the output is formatted similar to the provided urlencoded string. """ - return urllib.parse.urlencode(s, False, errors="surrogateescape") + + remove_trailing_equal = False + if similar_to: + remove_trailing_equal = any("=" not in param for param in similar_to.split("&")) + + encoded = urllib.parse.urlencode(s, False, errors="surrogateescape") + + if remove_trailing_equal: + encoded = encoded.replace("=&", "&") + if encoded[-1] == '=': + encoded = encoded[:-1] + + return encoded def decode(s): diff --git a/test/mitmproxy/net/http/test_url.py b/test/mitmproxy/net/http/test_url.py index 11ab1b817..2064aab8d 100644 --- a/test/mitmproxy/net/http/test_url.py +++ b/test/mitmproxy/net/http/test_url.py @@ -85,6 +85,26 @@ surrogates_quoted = ( ) +def test_empty_key_trailing_equal_sign(): + """ + Some HTTP clients don't send trailing equal signs for parameters without assigned value, e.g. they send + foo=bar&baz&qux=quux + instead of + foo=bar&baz=&qux=quux + The respective behavior of encode() should be driven by a reference string given in similar_to parameter + """ + reference_without_equal = "key1=val1&key2&key3=val3" + reference_with_equal = "key1=val1&key2=&key3=val3" + + post_data_empty_key_middle = [('one', 'two'), ('emptykey', ''), ('three', 'four')] + post_data_empty_key_end = [('one', 'two'), ('three', 'four'), ('emptykey', '')] + + assert url.encode(post_data_empty_key_middle, similar_to = reference_with_equal) == "one=two&emptykey=&three=four" + assert url.encode(post_data_empty_key_end, similar_to = reference_with_equal) == "one=two&three=four&emptykey=" + assert url.encode(post_data_empty_key_middle, similar_to = reference_without_equal) == "one=two&emptykey&three=four" + assert url.encode(post_data_empty_key_end, similar_to = reference_without_equal) == "one=two&three=four&emptykey" + + def test_encode(): assert url.encode([('foo', 'bar')]) assert url.encode([('foo', surrogates)])