From 4df325335b4abcdea6d59314ebfc96e7465a3979 Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Fri, 14 Dec 2018 21:31:34 +0530 Subject: [PATCH 1/4] multipart-fix --- mitmproxy/net/http/request.py | 3 ++- mitmproxy/tools/console/grideditor/editors.py | 12 ++++++++++-- test/mitmproxy/net/http/test_request.py | 5 +++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 959fdd339..218699e0d 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -468,7 +468,8 @@ class Request(message.Message): return () def _set_multipart_form(self, value): - raise NotImplementedError() + self.headers["content-type"] = "multipart/form-data" + self.content = mitmproxy.net.http.url.encode(value, self.get_text(strict=False)).encode() @property def multipart_form(self): diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index 61fcf6b45..21cc8159c 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -54,16 +54,24 @@ class ResponseHeaderEditor(HeaderEditor): class RequestFormEditor(base.FocusEditor): - title = "Edit URL-encoded Form" + title = "Edit Form" columns = [ col_text.Column("Key"), col_text.Column("Value") ] def get_data(self, flow): - return flow.request.urlencoded_form.items(multi=True) + + if "application/x-www-form-urlencoded" in flow.request.headers['Content-Type']: + return flow.request.urlencoded_form.items(multi=True) + + return flow.request.multipart_form.items(multi=True) def set_data(self, vals, flow): + + if "multipart/form-data" in flow.request.headers['Content-Type']: + flow.request.multipart_form = vals + flow.request.urlencoded_form = vals diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py index ef581a914..6ef73389b 100644 --- a/test/mitmproxy/net/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -372,5 +372,6 @@ class TestRequestUtils: def test_set_multipart_form(self): request = treq(content=b"foobar") - with pytest.raises(NotImplementedError): - request.multipart_form = "foobar" + request.multipart_form = [("filename", "shell.jpg"), ("file_size", "1000")] + assert request.headers['Content-Type'] == "multipart/form-data" + assert request.content From d08d2185eab0d58eef7a2b32d557475e51acb61a Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Tue, 29 Jan 2019 22:35:01 +0530 Subject: [PATCH 2/4] multipart encoder and tests --- mitmproxy/net/http/multipart.py | 55 ++++++++++++++++++----- mitmproxy/net/http/request.py | 2 +- test/mitmproxy/net/http/test_multipart.py | 18 ++++++++ test/mitmproxy/net/http/test_request.py | 8 ++-- 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/mitmproxy/net/http/multipart.py b/mitmproxy/net/http/multipart.py index a854d47fd..4edf76acd 100644 --- a/mitmproxy/net/http/multipart.py +++ b/mitmproxy/net/http/multipart.py @@ -1,8 +1,43 @@ import re - +import mimetypes +from urllib.parse import quote from mitmproxy.net.http import headers +def encode(head, l): + + k = head.get("content-type") + if k: + k = headers.parse_content_type(k) + if k is not None: + try: + boundary = k[2]["boundary"].encode("ascii") + boundary = quote(boundary) + except (KeyError, UnicodeError): + return b"" + hdrs = [] + for key, value in l: + file_type = mimetypes.guess_type(str(key))[0] or "text/plain; charset=utf-8" + + if key: + hdrs.append(b"--%b" % boundary.encode('utf-8')) + disposition = b'form-data; name="%b"' % key + hdrs.append(b"Content-Disposition: %b" % disposition) + hdrs.append(b"Content-Type: %b" % file_type.encode('utf-8')) + hdrs.append(b'') + hdrs.append(value) + hdrs.append(b'') + + if value is not None: + # If boundary is found in value then raise ValueError + if re.search(rb"^--%b$" % re.escape(boundary.encode('utf-8')), value): + raise ValueError(b"boundary found in encoded string") + + hdrs.append(b"--%b--\r\n" % boundary.encode('utf-8')) + temp = b"\r\n".join(hdrs) + return temp + + def decode(hdrs, content): """ Takes a multipart boundary encoded string and returns list of (key, value) tuples. @@ -19,14 +54,14 @@ def decode(hdrs, content): rx = re.compile(br'\bname="([^"]+)"') r = [] - - for i in content.split(b"--" + boundary): - parts = i.splitlines() - if len(parts) > 1 and parts[0][0:2] != b"--": - match = rx.search(parts[1]) - if match: - key = match.group(1) - value = b"".join(parts[3 + parts[2:].index(b""):]) - r.append((key, value)) + if content is not None: + for i in content.split(b"--" + boundary): + parts = i.splitlines() + if len(parts) > 1 and parts[0][0:2] != b"--": + match = rx.search(parts[1]) + if match: + key = match.group(1) + value = b"".join(parts[3 + parts[2:].index(b""):]) + r.append((key, value)) return r return [] diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 218699e0d..783fd5ff4 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -468,8 +468,8 @@ class Request(message.Message): return () def _set_multipart_form(self, value): + self.content = mitmproxy.net.http.multipart.encode(self.headers, value) self.headers["content-type"] = "multipart/form-data" - self.content = mitmproxy.net.http.url.encode(value, self.get_text(strict=False)).encode() @property def multipart_form(self): diff --git a/test/mitmproxy/net/http/test_multipart.py b/test/mitmproxy/net/http/test_multipart.py index 68ae6bbdf..ce7dee5a7 100644 --- a/test/mitmproxy/net/http/test_multipart.py +++ b/test/mitmproxy/net/http/test_multipart.py @@ -1,5 +1,6 @@ from mitmproxy.net.http import Headers from mitmproxy.net.http import multipart +import pytest def test_decode(): @@ -22,3 +23,20 @@ def test_decode(): assert len(form) == 2 assert form[0] == (b"field1", b"value1") assert form[1] == (b"field2", b"value2") + + +def test_encode(): + data = [("file".encode('utf-8'), "shell.jpg".encode('utf-8')), + ("file_size".encode('utf-8'), "1000".encode('utf-8'))] + headers = Headers( + content_type='multipart/form-data; boundary=127824672498' + ) + content = multipart.encode(headers, data) + + assert b'Content-Disposition: form-data; name="file"' in content + assert b'Content-Type: text/plain; charset=utf-8\r\n\r\nshell.jpg\r\n\r\n--127824672498\r\n' in content + assert b'1000\r\n\r\n--127824672498--\r\n' + assert len(content) == 252 + + with pytest.raises(ValueError, match=r"boundary found in encoded string"): + multipart.encode(headers, [("key".encode('utf-8'), "--127824672498".encode('utf-8'))]) diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py index 6ef73389b..71d5c7a12 100644 --- a/test/mitmproxy/net/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -371,7 +371,7 @@ class TestRequestUtils: assert list(request.multipart_form.items()) == [] def test_set_multipart_form(self): - request = treq(content=b"foobar") - request.multipart_form = [("filename", "shell.jpg"), ("file_size", "1000")] - assert request.headers['Content-Type'] == "multipart/form-data" - assert request.content + request = treq() + request.multipart_form = [("file", "shell.jpg"), ("file_size", "1000")] + assert request.headers["Content-Type"] == 'multipart/form-data' + assert request.content is None From 8948703470fdb4b46adf901b2b0918fd74c4df1f Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Wed, 30 Jan 2019 16:22:16 +0530 Subject: [PATCH 3/4] separate editors --- mitmproxy/tools/console/consoleaddons.py | 9 +++++--- mitmproxy/tools/console/grideditor/editors.py | 22 +++++++++++++------ mitmproxy/tools/console/window.py | 3 ++- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index a40cdeaa7..58f236c08 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -368,7 +368,8 @@ class ConsoleAddon: """ return [ "cookies", - "form", + "urlencoded form", + "multipart form", "path", "method", "query", @@ -403,8 +404,10 @@ class ConsoleAddon: flow.response = http.HTTPResponse.make() if part == "cookies": self.master.switch_view("edit_focus_cookies") - elif part == "form": - self.master.switch_view("edit_focus_form") + elif part == "urlencoded form": + self.master.switch_view("edit_focus_urlencoded_form") + elif part == "multipart form": + self.master.switch_view("edit_focus_multipart_form") elif part == "path": self.master.switch_view("edit_focus_path") elif part == "query": diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index 21cc8159c..09666d588 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -53,8 +53,8 @@ class ResponseHeaderEditor(HeaderEditor): flow.response.headers = Headers(vals) -class RequestFormEditor(base.FocusEditor): - title = "Edit Form" +class RequestMultipartEditor(base.FocusEditor): + title = "Edit Multipart Form" columns = [ col_text.Column("Key"), col_text.Column("Value") @@ -62,16 +62,24 @@ class RequestFormEditor(base.FocusEditor): def get_data(self, flow): - if "application/x-www-form-urlencoded" in flow.request.headers['Content-Type']: - return flow.request.urlencoded_form.items(multi=True) - return flow.request.multipart_form.items(multi=True) def set_data(self, vals, flow): + flow.request.multipart_form = vals - if "multipart/form-data" in flow.request.headers['Content-Type']: - flow.request.multipart_form = vals +class RequestUrlEncodedEditor(base.FocusEditor): + title = "Edit UrlEncoded Form" + columns = [ + col_text.Column("Key"), + col_text.Column("Value") + ] + + def get_data(self, flow): + + return flow.request.urlencoded_form.items(multi=True) + + def set_data(self, vals, flow): flow.request.urlencoded_form = vals diff --git a/mitmproxy/tools/console/window.py b/mitmproxy/tools/console/window.py index 7669299c7..fb2e8c1e6 100644 --- a/mitmproxy/tools/console/window.py +++ b/mitmproxy/tools/console/window.py @@ -64,7 +64,8 @@ class WindowStack: edit_focus_cookies = grideditor.CookieEditor(master), edit_focus_setcookies = grideditor.SetCookieEditor(master), edit_focus_setcookie_attrs = grideditor.CookieAttributeEditor(master), - edit_focus_form = grideditor.RequestFormEditor(master), + edit_focus_multipart_form=grideditor.RequestMultipartEditor(master), + edit_focus_urlencoded_form=grideditor.RequestUrlEncodedEditor(master), edit_focus_path = grideditor.PathEditor(master), edit_focus_request_headers = grideditor.RequestHeaderEditor(master), edit_focus_response_headers = grideditor.ResponseHeaderEditor(master), From 580ba356adf4e11241725005eb79d47f3468e092 Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Wed, 6 Feb 2019 03:35:36 +0530 Subject: [PATCH 4/4] test coverage improved --- test/mitmproxy/net/http/test_multipart.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/mitmproxy/net/http/test_multipart.py b/test/mitmproxy/net/http/test_multipart.py index ce7dee5a7..6d2e50170 100644 --- a/test/mitmproxy/net/http/test_multipart.py +++ b/test/mitmproxy/net/http/test_multipart.py @@ -24,6 +24,18 @@ def test_decode(): assert form[0] == (b"field1", b"value1") assert form[1] == (b"field2", b"value2") + boundary = 'boundary茅莽' + headers = Headers( + content_type='multipart/form-data; boundary=' + boundary + ) + result = multipart.decode(headers, content) + assert result == [] + + headers = Headers( + content_type='' + ) + assert multipart.decode(headers, content) == [] + def test_encode(): data = [("file".encode('utf-8'), "shell.jpg".encode('utf-8')), @@ -40,3 +52,10 @@ def test_encode(): with pytest.raises(ValueError, match=r"boundary found in encoded string"): multipart.encode(headers, [("key".encode('utf-8'), "--127824672498".encode('utf-8'))]) + + boundary = 'boundary茅莽' + headers = Headers( + content_type='multipart/form-data; boundary=' + boundary + ) + result = multipart.encode(headers, data) + assert result == b''