diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 90c8bd89d..9375f973b 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -293,16 +293,20 @@ class ConsoleMaster(flow.FlowMaster): - a list of flows, otherwise. """ try: - return flow.read_flows_from_paths([path]) + return flow.read_flows_from_paths(path) except flow.FlowReadError as e: signals.status_message.send(message=e.strerror) def client_playback_path(self, path): + if not isinstance(path, list): + path = [path] flows = self._readflows(path) if flows: self.start_client_playback(flows, False) def server_playback_path(self, path): + if not isinstance(path, list): + path = [path] flows = self._readflows(path) if flows: self.start_server_playback( diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 2f143f013..bc8a2aad8 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -252,7 +252,7 @@ def copy_flow(part, scope, flow, master, state): try: master.add_event(str(len(data))) pyperclip.copy(data) - except RuntimeError: + except (RuntimeError, UnicodeDecodeError): def save(k): if k == "y": ask_save_path("Save data", data, master, state) diff --git a/libmproxy/console/contentview.py b/libmproxy/console/contentview.py index 95d908a45..12ed5b64b 100644 --- a/libmproxy/console/contentview.py +++ b/libmproxy/console/contentview.py @@ -240,33 +240,13 @@ class ViewMultipart: content_types = ["multipart/form-data"] def __call__(self, hdrs, content, limit): - v = hdrs.get_first("content-type") + v = utils.multipartdecode(hdrs, content) if v: - v = utils.parse_content_type(v) - if not v: - return - boundary = v[2].get("boundary") - if not boundary: - return - - rx = re.compile(r'\bname="([^"]+)"') - keys = [] - vals = [] - - for i in content.split("--" + boundary): - parts = i.splitlines() - if len(parts) > 1 and parts[0][0:2] != "--": - match = rx.search(parts[1]) - if match: - keys.append(match.group(1) + ":") - vals.append(netlib.utils.cleanBin( - "\n".join(parts[3+parts[2:].index(""):]) - )) r = [ urwid.Text(("highlight", "Form data:\n")), ] r.extend(common.format_keyvals( - zip(keys, vals), + v, key = "header", val = "text" )) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 9aa2a179f..eb8ee16a6 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -242,7 +242,7 @@ class ServerPlaybackState: ] if not self.ignore_content: - form_contents = r.get_form_urlencoded() + form_contents = r.get_form() if self.ignore_payload_params and form_contents: key.extend( p for p in form_contents @@ -775,6 +775,8 @@ class FlowMaster(controller.Master): if all(e): self.shutdown() self.client_playback.tick(self) + if self.client_playback.done(): + self.client_playback = None return super(FlowMaster, self).tick(q, timeout) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 00086c211..c763db4c6 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -15,6 +15,7 @@ from ..proxy.connection import ServerConnection from .. import encoding, utils, controller, stateobject, proxy HDR_FORM_URLENCODED = "application/x-www-form-urlencoded" +HDR_FORM_MULTIPART = "multipart/form-data" CONTENT_MISSING = 0 @@ -509,6 +510,19 @@ class HTTPRequest(HTTPMessage): """ self.headers["Host"] = [self.host] + def get_form(self): + """ + Retrieves the URL-encoded or multipart form data, returning an ODict object. + Returns an empty ODict if there is no data or the content-type + indicates non-form data. + """ + if self.content: + if self.headers.in_any("content-type", HDR_FORM_URLENCODED, True): + return self.get_form_urlencoded() + elif self.headers.in_any("content-type", HDR_FORM_MULTIPART, True): + return self.get_form_multipart() + return ODict([]) + def get_form_urlencoded(self): """ Retrieves the URL-encoded form data, returning an ODict object. @@ -516,7 +530,12 @@ class HTTPRequest(HTTPMessage): indicates non-form data. """ if self.content and self.headers.in_any("content-type", HDR_FORM_URLENCODED, True): - return ODict(utils.urldecode(self.content)) + return ODict(utils.urldecode(self.content)) + return ODict([]) + + def get_form_multipart(self): + if self.content and self.headers.in_any("content-type", HDR_FORM_MULTIPART, True): + return ODict(utils.multipartdecode(self.headers, self.content)) return ODict([]) def set_form_urlencoded(self, odict): diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 5ed70a45e..02e8403bf 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -69,6 +69,33 @@ def urlencode(s): return urllib.urlencode(s, False) +def multipartdecode(hdrs, content): + """ + Takes a multipart boundary encoded string and returns list of (key, value) tuples. + """ + v = hdrs.get_first("content-type") + if v: + v = parse_content_type(v) + if not v: + return [] + boundary = v[2].get("boundary") + if not boundary: + return [] + + rx = re.compile(r'\bname="([^"]+)"') + r = [] + + for i in content.split("--" + boundary): + parts = i.splitlines() + if len(parts) > 1 and parts[0][0:2] != "--": + match = rx.search(parts[1]) + if match: + key = match.group(1) + value = "".join(parts[3+parts[2:].index(""):]) + r.append((key, value)) + return r + return [] + def pretty_size(size): suffixes = [ ("B", 2**10), diff --git a/libmproxy/web/__init__.py b/libmproxy/web/__init__.py index 173ddf9f5..7016eeb8c 100644 --- a/libmproxy/web/__init__.py +++ b/libmproxy/web/__init__.py @@ -133,6 +133,9 @@ class WebMaster(flow.FlowMaster): "Could not read flow file: %s"%v, "error" ) + if self.options.app: + self.start_app(self.options.app_host, self.options.app_port) + def tick(self): flow.FlowMaster.tick(self, self.masterq, timeout=0) @@ -173,4 +176,4 @@ class WebMaster(flow.FlowMaster): def add_event(self, e, level="info"): super(WebMaster, self).add_event(e, level) - self.state.add_event(e, level) \ No newline at end of file + self.state.add_event(e, level) diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index 16870777d..23c3f469b 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -1,3 +1,4 @@ +from mock import MagicMock from libmproxy.protocol.http import * from cStringIO import StringIO import tutils, tservers @@ -112,6 +113,26 @@ class TestHTTPRequest: r = tutils.treq() assert repr(r) + def test_get_form_for_urlencoded(self): + r = tutils.treq() + r.headers.add("content-type", "application/x-www-form-urlencoded") + r.get_form_urlencoded = MagicMock() + + r.get_form() + + assert r.get_form_urlencoded.called + + def test_get_form_for_multipart(self): + r = tutils.treq() + r.headers.add("content-type", "multipart/form-data") + r.get_form_multipart = MagicMock() + + r.get_form() + + assert r.get_form_multipart.called + + + class TestHTTPResponse: def test_read_from_stringio(self): diff --git a/test/test_utils.py b/test/test_utils.py index 1678a7dee..35ba0c9d5 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,5 +1,5 @@ import json -from libmproxy import utils +from libmproxy import utils, flow import tutils utils.CERT_SLEEP_TIME = 0 @@ -52,6 +52,23 @@ def test_urldecode(): s = "one=two&three=four" assert len(utils.urldecode(s)) == 2 +def test_multipartdecode(): + boundary = 'somefancyboundary' + headers = flow.ODict([('content-type', ('multipart/form-data; boundary=%s' % boundary))]) + content = "--{0}\n" \ + "Content-Disposition: form-data; name=\"field1\"\n\n" \ + "value1\n" \ + "--{0}\n" \ + "Content-Disposition: form-data; name=\"field2\"\n\n" \ + "value2\n" \ + "--{0}--".format(boundary) + + form = utils.multipartdecode(headers, content) + + assert len(form) == 2 + assert form[0] == ('field1', 'value1') + assert form[1] == ('field2', 'value2') + def test_pretty_duration(): assert utils.pretty_duration(0.00001) == "0ms" assert utils.pretty_duration(0.0001) == "0ms"