From 244ef243d75145a01d9029589de65be51299b3f3 Mon Sep 17 00:00:00 2001 From: Krzysztof Bielicki Date: Tue, 10 Mar 2015 10:44:06 +0100 Subject: [PATCH 1/8] [#514] Add support for ignoring payload params in multipart/form-data --- libmproxy/console/contentview.py | 24 ++---------------------- libmproxy/flow.py | 2 +- libmproxy/protocol/http.py | 21 ++++++++++++++++++++- libmproxy/utils.py | 27 +++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 24 deletions(-) diff --git a/libmproxy/console/contentview.py b/libmproxy/console/contentview.py index 582723bb1..84e9946d9 100644 --- a/libmproxy/console/contentview.py +++ b/libmproxy/console/contentview.py @@ -210,33 +210,13 @@ class ViewMultipart: prompt = ("multipart", "m") content_types = ["multipart/form-data"] def __call__(self, hdrs, content, limit): - v = hdrs.get_first("content-type") + v = utils.multipartdecode(hdrs, content) if v: - v = utils.parse_content_type(v) - if not v: - return - boundary = v[2].get("boundary") - if not boundary: - return - - rx = re.compile(r'\bname="([^"]+)"') - keys = [] - vals = [] - - for i in content.split("--" + boundary): - parts = i.splitlines() - if len(parts) > 1 and parts[0][0:2] != "--": - match = rx.search(parts[1]) - if match: - keys.append(match.group(1) + ":") - vals.append(netlib.utils.cleanBin( - "\n".join(parts[3+parts[2:].index(""):]) - )) r = [ urwid.Text(("highlight", "Form data:\n")), ] r.extend(common.format_keyvals( - zip(keys, vals), + v, key = "header", val = "text" )) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 43580109c..0e9e481c8 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -236,7 +236,7 @@ class ServerPlaybackState: ] if not self.ignore_content: - form_contents = r.get_form_urlencoded() + form_contents = r.get_form() if self.ignore_payload_params and form_contents: key.extend( p for p in form_contents diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 49310ec3f..512cf75b1 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -15,6 +15,7 @@ from ..proxy.connection import ServerConnection from .. import encoding, utils, controller, stateobject, proxy HDR_FORM_URLENCODED = "application/x-www-form-urlencoded" +HDR_FORM_MULTIPART = "multipart/form-data" CONTENT_MISSING = 0 @@ -507,6 +508,19 @@ class HTTPRequest(HTTPMessage): """ self.headers["Host"] = [self.host] + def get_form(self): + """ + Retrieves the URL-encoded or multipart form data, returning an ODict object. + Returns an empty ODict if there is no data or the content-type + indicates non-form data. + """ + if self.content: + if self.headers.in_any("content-type", HDR_FORM_URLENCODED, True): + return self.get_form_urlencoded() + elif self.headers.in_any("content-type", HDR_FORM_MULTIPART, True): + return self.get_form_multipart() + return ODict([]) + def get_form_urlencoded(self): """ Retrieves the URL-encoded form data, returning an ODict object. @@ -514,7 +528,12 @@ class HTTPRequest(HTTPMessage): indicates non-form data. """ if self.content and self.headers.in_any("content-type", HDR_FORM_URLENCODED, True): - return ODict(utils.urldecode(self.content)) + return ODict(utils.urldecode(self.content)) + return ODict([]) + + def get_form_multipart(self): + if self.content and self.headers.in_any("content-type", HDR_FORM_MULTIPART, True): + return ODict(utils.multipartdecode(self.headers, self.content)) return ODict([]) def set_form_urlencoded(self, odict): diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 51f2dc26a..b84c589a4 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -69,6 +69,33 @@ def urlencode(s): return urllib.urlencode(s, False) +def multipartdecode(hdrs, content): + """ + Takes a multipart boundary encoded string and returns list of (key, value) tuples. + """ + v = hdrs.get_first("content-type") + if v: + v = parse_content_type(v) + if not v: + return [] + boundary = v[2].get("boundary") + if not boundary: + return [] + + rx = re.compile(r'\bname="([^"]+)"') + r = [] + + for i in content.split("--" + boundary): + parts = i.splitlines() + if len(parts) > 1 and parts[0][0:2] != "--": + match = rx.search(parts[1]) + if match: + key = match.group(1) + value = "".join(parts[3+parts[2:].index(""):]) + r.append((key, value)) + return r + return [] + def pretty_size(size): suffixes = [ ("B", 2**10), From 953f9aa64166451a07502f05c15db47c053e6081 Mon Sep 17 00:00:00 2001 From: Krzysztof Bielicki Date: Mon, 16 Mar 2015 10:23:50 +0100 Subject: [PATCH 2/8] Added tests --- test/test_protocol_http.py | 21 +++++++++++++++++++++ test/test_utils.py | 19 ++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index 16870777d..23c3f469b 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -1,3 +1,4 @@ +from mock import MagicMock from libmproxy.protocol.http import * from cStringIO import StringIO import tutils, tservers @@ -112,6 +113,26 @@ class TestHTTPRequest: r = tutils.treq() assert repr(r) + def test_get_form_for_urlencoded(self): + r = tutils.treq() + r.headers.add("content-type", "application/x-www-form-urlencoded") + r.get_form_urlencoded = MagicMock() + + r.get_form() + + assert r.get_form_urlencoded.called + + def test_get_form_for_multipart(self): + r = tutils.treq() + r.headers.add("content-type", "multipart/form-data") + r.get_form_multipart = MagicMock() + + r.get_form() + + assert r.get_form_multipart.called + + + class TestHTTPResponse: def test_read_from_stringio(self): diff --git a/test/test_utils.py b/test/test_utils.py index 78d1c0725..a79029108 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,5 +1,5 @@ import json -from libmproxy import utils +from libmproxy import utils, flow import tutils utils.CERT_SLEEP_TIME = 0 @@ -52,6 +52,23 @@ def test_urldecode(): s = "one=two&three=four" assert len(utils.urldecode(s)) == 2 +def test_multipartdecode(): + boundary = 'somefancyboundary' + headers = flow.ODict([('content-type', ('multipart/form-data; boundary=%s' % boundary))]) + content = "--{0}\n" \ + "Content-Disposition: form-data; name=\"field1\"\n\n" \ + "value1\n" \ + "--{0}\n" \ + "Content-Disposition: form-data; name=\"field2\"\n\n" \ + "value2\n" \ + "--{0}--".format(boundary) + + form = utils.multipartdecode(headers, content) + + assert len(form) == 2 + assert form[0] == ('field1', 'value1') + assert form[1] == ('field2', 'value2') + def test_pretty_duration(): assert utils.pretty_duration(0.00001) == "0ms" assert utils.pretty_duration(0.0001) == "0ms" From f45ac12d203a9ace1879ed5fa164f2890f9ce207 Mon Sep 17 00:00:00 2001 From: Marcelo Glezer Date: Mon, 23 Mar 2015 23:57:18 -0300 Subject: [PATCH 3/8] handles UnicodeDecodeError --- libmproxy/console/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 2f143f013..bc8a2aad8 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -252,7 +252,7 @@ def copy_flow(part, scope, flow, master, state): try: master.add_event(str(len(data))) pyperclip.copy(data) - except RuntimeError: + except (RuntimeError, UnicodeDecodeError): def save(k): if k == "y": ask_save_path("Save data", data, master, state) From e6a8863c51f2e2f5809ae0a42fc874afc3c7f908 Mon Sep 17 00:00:00 2001 From: Marcelo Glezer Date: Wed, 25 Mar 2015 16:57:28 -0300 Subject: [PATCH 4/8] fixed exception in mitmproxy -c fix #535 --- libmproxy/console/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 90c8bd89d..de549ee6c 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -293,7 +293,7 @@ class ConsoleMaster(flow.FlowMaster): - a list of flows, otherwise. """ try: - return flow.read_flows_from_paths([path]) + return flow.read_flows_from_paths(path) except flow.FlowReadError as e: signals.status_message.send(message=e.strerror) From c5f153f8d0e50e12b9c9cd6ab0988a1294bcd4be Mon Sep 17 00:00:00 2001 From: Marcelo Glezer Date: Wed, 25 Mar 2015 17:07:31 -0300 Subject: [PATCH 5/8] fix #535 now works with both cases: using -c from command line and pressing c in the ui --- libmproxy/console/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index de549ee6c..75d1e1ace 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -298,6 +298,8 @@ class ConsoleMaster(flow.FlowMaster): signals.status_message.send(message=e.strerror) def client_playback_path(self, path): + if not isinstance(path, list): + path = [path] flows = self._readflows(path) if flows: self.start_client_playback(flows, False) From fb17eea5e0adcbe8d0dc5c2f029b10b172d5a3e7 Mon Sep 17 00:00:00 2001 From: Marcelo Glezer Date: Wed, 25 Mar 2015 17:12:38 -0300 Subject: [PATCH 6/8] fix #535 server replay was failing from ui but working from command line (fixed now too) --- libmproxy/console/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 75d1e1ace..9375f973b 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -305,6 +305,8 @@ class ConsoleMaster(flow.FlowMaster): self.start_client_playback(flows, False) def server_playback_path(self, path): + if not isinstance(path, list): + path = [path] flows = self._readflows(path) if flows: self.start_server_playback( From ab7281ec0452049ff053128ba741afbd9d1cfdb3 Mon Sep 17 00:00:00 2001 From: Marcelo Glezer Date: Wed, 25 Mar 2015 17:40:25 -0300 Subject: [PATCH 7/8] fix #530 set client_playback to None when client_playback is done --- libmproxy/flow.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 8343c1832..bdf49133e 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -771,6 +771,8 @@ class FlowMaster(controller.Master): if all(e): self.shutdown() self.client_playback.tick(self) + if self.client_playback.done(): + self.client_playback = None return super(FlowMaster, self).tick(q, timeout) From 901bf018c06bb75ae35106b60a7553bfed25195a Mon Sep 17 00:00:00 2001 From: peralta Date: Thu, 26 Mar 2015 17:20:32 +0100 Subject: [PATCH 8/8] mitmweb: start onboarding app if instructed to --- libmproxy/web/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libmproxy/web/__init__.py b/libmproxy/web/__init__.py index 173ddf9f5..7016eeb8c 100644 --- a/libmproxy/web/__init__.py +++ b/libmproxy/web/__init__.py @@ -133,6 +133,9 @@ class WebMaster(flow.FlowMaster): "Could not read flow file: %s"%v, "error" ) + if self.options.app: + self.start_app(self.options.app_host, self.options.app_port) + def tick(self): flow.FlowMaster.tick(self, self.masterq, timeout=0) @@ -173,4 +176,4 @@ class WebMaster(flow.FlowMaster): def add_event(self, e, level="info"): super(WebMaster, self).add_event(e, level) - self.state.add_event(e, level) \ No newline at end of file + self.state.add_event(e, level)