Merge branch 'master' of github.com:mitmproxy/mitmproxy

libmproxy/console/__init__.py

@@ -293,16 +293,20 @@ class ConsoleMaster(flow.FlowMaster):
             - a list of flows, otherwise.
         """
         try:
-            return flow.read_flows_from_paths([path])
+            return flow.read_flows_from_paths(path)
         except flow.FlowReadError as e:
             signals.status_message.send(message=e.strerror)
 
     def client_playback_path(self, path):
+        if not isinstance(path, list):
+            path = [path]
         flows = self._readflows(path)
         if flows:
             self.start_client_playback(flows, False)
 
     def server_playback_path(self, path):
+        if not isinstance(path, list):
+            path = [path]
         flows = self._readflows(path)
         if flows:
             self.start_server_playback(

libmproxy/console/common.py

@@ -252,7 +252,7 @@ def copy_flow(part, scope, flow, master, state):
     try:
         master.add_event(str(len(data)))
         pyperclip.copy(data)
-    except RuntimeError:
+    except (RuntimeError, UnicodeDecodeError):
         def save(k):
             if k == "y":
                 ask_save_path("Save data", data, master, state)

libmproxy/console/contentview.py

@@ -240,33 +240,13 @@ class ViewMultipart:
     content_types = ["multipart/form-data"]
 
     def __call__(self, hdrs, content, limit):
-        v = hdrs.get_first("content-type")
+        v = utils.multipartdecode(hdrs, content)
         if v:
-            v = utils.parse_content_type(v)
-            if not v:
-                return
-            boundary = v[2].get("boundary")
-            if not boundary:
-                return
-
-            rx = re.compile(r'\bname="([^"]+)"')
-            keys = []
-            vals = []
-
-            for i in content.split("--" + boundary):
-                parts = i.splitlines()
-                if len(parts) > 1 and parts[0][0:2] != "--":
-                    match = rx.search(parts[1])
-                    if match:
-                        keys.append(match.group(1) + ":")
-                        vals.append(netlib.utils.cleanBin(
-                            "\n".join(parts[3+parts[2:].index(""):])
-                        ))
             r = [
                 urwid.Text(("highlight", "Form data:\n")),
             ]
             r.extend(common.format_keyvals(
-                zip(keys, vals),
+                v,
                 key = "header",
                 val = "text"
             ))

libmproxy/flow.py

@@ -242,7 +242,7 @@ class ServerPlaybackState:
         ]
 
         if not self.ignore_content:
-            form_contents = r.get_form_urlencoded()
+            form_contents = r.get_form()
             if self.ignore_payload_params and form_contents:
                 key.extend(
                     p for p in form_contents
@@ -775,6 +775,8 @@ class FlowMaster(controller.Master):
             if all(e):
                 self.shutdown()
             self.client_playback.tick(self)
+            if self.client_playback.done():
+                self.client_playback = None
 
         return super(FlowMaster, self).tick(q, timeout)
 

libmproxy/protocol/http.py

@@ -15,6 +15,7 @@ from ..proxy.connection import ServerConnection
 from .. import encoding, utils, controller, stateobject, proxy
 
 HDR_FORM_URLENCODED = "application/x-www-form-urlencoded"
+HDR_FORM_MULTIPART = "multipart/form-data"
 CONTENT_MISSING = 0
 
 
@@ -509,6 +510,19 @@ class HTTPRequest(HTTPMessage):
         """
         self.headers["Host"] = [self.host]
 
+    def get_form(self):
+        """
+            Retrieves the URL-encoded or multipart form data, returning an ODict object.
+            Returns an empty ODict if there is no data or the content-type
+            indicates non-form data.
+        """
+        if self.content:
+            if self.headers.in_any("content-type", HDR_FORM_URLENCODED, True):
+                return self.get_form_urlencoded()
+            elif self.headers.in_any("content-type", HDR_FORM_MULTIPART, True):
+                return self.get_form_multipart()
+        return ODict([])
+
     def get_form_urlencoded(self):
         """
             Retrieves the URL-encoded form data, returning an ODict object.
@@ -516,7 +530,12 @@ class HTTPRequest(HTTPMessage):
             indicates non-form data.
         """
         if self.content and self.headers.in_any("content-type", HDR_FORM_URLENCODED, True):
-            return ODict(utils.urldecode(self.content))
+                return ODict(utils.urldecode(self.content))
+        return ODict([])
+
+    def get_form_multipart(self):
+        if self.content and self.headers.in_any("content-type", HDR_FORM_MULTIPART, True):
+                return ODict(utils.multipartdecode(self.headers, self.content))
         return ODict([])
 
     def set_form_urlencoded(self, odict):

libmproxy/utils.py

@@ -69,6 +69,33 @@ def urlencode(s):
     return urllib.urlencode(s, False)
 
 
+def multipartdecode(hdrs, content):
+    """
+        Takes a multipart boundary encoded string and returns list of (key, value) tuples.
+    """
+    v = hdrs.get_first("content-type")
+    if v:
+        v = parse_content_type(v)
+        if not v:
+            return []
+        boundary = v[2].get("boundary")
+        if not boundary:
+            return []
+
+        rx = re.compile(r'\bname="([^"]+)"')
+        r = []
+
+        for i in content.split("--" + boundary):
+            parts = i.splitlines()
+            if len(parts) > 1 and parts[0][0:2] != "--":
+                match = rx.search(parts[1])
+                if match:
+                    key = match.group(1)
+                    value = "".join(parts[3+parts[2:].index(""):])
+                    r.append((key, value))
+        return r
+    return []
+
 def pretty_size(size):
     suffixes = [
         ("B",   2**10),

libmproxy/web/__init__.py

@@ -133,6 +133,9 @@ class WebMaster(flow.FlowMaster):
                     "Could not read flow file: %s"%v,
                     "error"
                 )
+        if self.options.app:
+            self.start_app(self.options.app_host, self.options.app_port)
+
 
     def tick(self):
         flow.FlowMaster.tick(self, self.masterq, timeout=0)
@@ -173,4 +176,4 @@ class WebMaster(flow.FlowMaster):
 
     def add_event(self, e, level="info"):
         super(WebMaster, self).add_event(e, level)
-        self.state.add_event(e, level)
+        self.state.add_event(e, level)

test/test_protocol_http.py

@@ -1,3 +1,4 @@
+from mock import MagicMock
 from libmproxy.protocol.http import *
 from cStringIO import StringIO
 import tutils, tservers
@@ -112,6 +113,26 @@ class TestHTTPRequest:
         r = tutils.treq()
         assert repr(r)
 
+    def test_get_form_for_urlencoded(self):
+        r = tutils.treq()
+        r.headers.add("content-type", "application/x-www-form-urlencoded")
+        r.get_form_urlencoded = MagicMock()
+
+        r.get_form()
+
+        assert r.get_form_urlencoded.called
+
+    def test_get_form_for_multipart(self):
+        r = tutils.treq()
+        r.headers.add("content-type", "multipart/form-data")
+        r.get_form_multipart = MagicMock()
+
+        r.get_form()
+
+        assert r.get_form_multipart.called
+
+
+
 
 class TestHTTPResponse:
     def test_read_from_stringio(self):

test/test_utils.py

@@ -1,5 +1,5 @@
 import json
-from libmproxy import utils
+from libmproxy import utils, flow
 import tutils
 
 utils.CERT_SLEEP_TIME = 0
@@ -52,6 +52,23 @@ def test_urldecode():
     s = "one=two&three=four"
     assert len(utils.urldecode(s)) == 2
 
+def test_multipartdecode():
+    boundary = 'somefancyboundary'
+    headers = flow.ODict([('content-type', ('multipart/form-data; boundary=%s' % boundary))])
+    content = "--{0}\n" \
+              "Content-Disposition: form-data; name=\"field1\"\n\n" \
+              "value1\n" \
+              "--{0}\n" \
+              "Content-Disposition: form-data; name=\"field2\"\n\n" \
+              "value2\n" \
+              "--{0}--".format(boundary)
+
+    form = utils.multipartdecode(headers, content)
+
+    assert len(form) == 2
+    assert form[0] == ('field1', 'value1')
+    assert form[1] == ('field2', 'value2')
+
 def test_pretty_duration():
     assert utils.pretty_duration(0.00001) == "0ms"
     assert utils.pretty_duration(0.0001) == "0ms"