Merge pull request #515 from krzysiekbielicki/master

[#514] Add support for ignoring payload params in multipart/form-data

libmproxy/console/contentview.py

@@ -240,33 +240,13 @@ class ViewMultipart:
     content_types = ["multipart/form-data"]
 
     def __call__(self, hdrs, content, limit):
-        v = hdrs.get_first("content-type")
+        v = utils.multipartdecode(hdrs, content)
         if v:
-            v = utils.parse_content_type(v)
-            if not v:
-                return
-            boundary = v[2].get("boundary")
-            if not boundary:
-                return
-
-            rx = re.compile(r'\bname="([^"]+)"')
-            keys = []
-            vals = []
-
-            for i in content.split("--" + boundary):
-                parts = i.splitlines()
-                if len(parts) > 1 and parts[0][0:2] != "--":
-                    match = rx.search(parts[1])
-                    if match:
-                        keys.append(match.group(1) + ":")
-                        vals.append(netlib.utils.cleanBin(
-                            "\n".join(parts[3+parts[2:].index(""):])
-                        ))
             r = [
                 urwid.Text(("highlight", "Form data:\n")),
             ]
             r.extend(common.format_keyvals(
-                zip(keys, vals),
+                v,
                 key = "header",
                 val = "text"
             ))

libmproxy/flow.py

@@ -242,7 +242,7 @@ class ServerPlaybackState:
         ]
 
         if not self.ignore_content:
-            form_contents = r.get_form_urlencoded()
+            form_contents = r.get_form()
             if self.ignore_payload_params and form_contents:
                 key.extend(
                     p for p in form_contents

libmproxy/protocol/http.py

@@ -15,6 +15,7 @@ from ..proxy.connection import ServerConnection
 from .. import encoding, utils, controller, stateobject, proxy
 
 HDR_FORM_URLENCODED = "application/x-www-form-urlencoded"
+HDR_FORM_MULTIPART = "multipart/form-data"
 CONTENT_MISSING = 0
 
 
@@ -509,6 +510,19 @@ class HTTPRequest(HTTPMessage):
         """
         self.headers["Host"] = [self.host]
 
+    def get_form(self):
+        """
+            Retrieves the URL-encoded or multipart form data, returning an ODict object.
+            Returns an empty ODict if there is no data or the content-type
+            indicates non-form data.
+        """
+        if self.content:
+            if self.headers.in_any("content-type", HDR_FORM_URLENCODED, True):
+                return self.get_form_urlencoded()
+            elif self.headers.in_any("content-type", HDR_FORM_MULTIPART, True):
+                return self.get_form_multipart()
+        return ODict([])
+
     def get_form_urlencoded(self):
         """
             Retrieves the URL-encoded form data, returning an ODict object.
@@ -519,6 +533,11 @@ class HTTPRequest(HTTPMessage):
                 return ODict(utils.urldecode(self.content))
         return ODict([])
 
+    def get_form_multipart(self):
+        if self.content and self.headers.in_any("content-type", HDR_FORM_MULTIPART, True):
+                return ODict(utils.multipartdecode(self.headers, self.content))
+        return ODict([])
+
     def set_form_urlencoded(self, odict):
         """
             Sets the body to the URL-encoded form data, and adds the

libmproxy/utils.py

@@ -69,6 +69,33 @@ def urlencode(s):
     return urllib.urlencode(s, False)
 
 
+def multipartdecode(hdrs, content):
+    """
+        Takes a multipart boundary encoded string and returns list of (key, value) tuples.
+    """
+    v = hdrs.get_first("content-type")
+    if v:
+        v = parse_content_type(v)
+        if not v:
+            return []
+        boundary = v[2].get("boundary")
+        if not boundary:
+            return []
+
+        rx = re.compile(r'\bname="([^"]+)"')
+        r = []
+
+        for i in content.split("--" + boundary):
+            parts = i.splitlines()
+            if len(parts) > 1 and parts[0][0:2] != "--":
+                match = rx.search(parts[1])
+                if match:
+                    key = match.group(1)
+                    value = "".join(parts[3+parts[2:].index(""):])
+                    r.append((key, value))
+        return r
+    return []
+
 def pretty_size(size):
     suffixes = [
         ("B",   2**10),

test/test_protocol_http.py

@@ -1,3 +1,4 @@
+from mock import MagicMock
 from libmproxy.protocol.http import *
 from cStringIO import StringIO
 import tutils, tservers
@@ -112,6 +113,26 @@ class TestHTTPRequest:
         r = tutils.treq()
         assert repr(r)
 
+    def test_get_form_for_urlencoded(self):
+        r = tutils.treq()
+        r.headers.add("content-type", "application/x-www-form-urlencoded")
+        r.get_form_urlencoded = MagicMock()
+
+        r.get_form()
+
+        assert r.get_form_urlencoded.called
+
+    def test_get_form_for_multipart(self):
+        r = tutils.treq()
+        r.headers.add("content-type", "multipart/form-data")
+        r.get_form_multipart = MagicMock()
+
+        r.get_form()
+
+        assert r.get_form_multipart.called
+
+
+
 
 class TestHTTPResponse:
     def test_read_from_stringio(self):

test/test_utils.py

@@ -1,5 +1,5 @@
 import json
-from libmproxy import utils
+from libmproxy import utils, flow
 import tutils
 
 utils.CERT_SLEEP_TIME = 0
@@ -52,6 +52,23 @@ def test_urldecode():
     s = "one=two&three=four"
     assert len(utils.urldecode(s)) == 2
 
+def test_multipartdecode():
+    boundary = 'somefancyboundary'
+    headers = flow.ODict([('content-type', ('multipart/form-data; boundary=%s' % boundary))])
+    content = "--{0}\n" \
+              "Content-Disposition: form-data; name=\"field1\"\n\n" \
+              "value1\n" \
+              "--{0}\n" \
+              "Content-Disposition: form-data; name=\"field2\"\n\n" \
+              "value2\n" \
+              "--{0}--".format(boundary)
+
+    form = utils.multipartdecode(headers, content)
+
+    assert len(form) == 2
+    assert form[0] == ('field1', 'value1')
+    assert form[1] == ('field2', 'value2')
+
 def test_pretty_duration():
     assert utils.pretty_duration(0.00001) == "0ms"
     assert utils.pretty_duration(0.0001) == "0ms"