Merge branch 'multi-cookie' of https://github.com/dufferzafar/mitmproxy

netlib/http/cookies.py

@@ -14,12 +14,9 @@ information. Duplicate cookies are preserved in parsing, and can be set in
 formatting. We do attempt to escape and quote values where needed, but will not
 reject data that violate the specs.
 
-Parsing accepts the formats in RFC6265 and partially RFC2109 and RFC2965. We do
+Parsing accepts the formats in RFC6265 and partially RFC2109 and RFC2965. We
-not parse the comma-separated variant of Set-Cookie that allows multiple
+also parse the comma-separated variant of Set-Cookie that allows multiple
-cookies to be set in a single header. Technically this should be feasible, but
+cookies to be set in a single header. Serialization follows RFC6265.
-it turns out that violations of RFC6265 that makes the parsing problem
-indeterminate are much more common than genuine occurences of the multi-cookie
-variants. Serialization follows RFC6265.
 
     http://tools.ietf.org/html/rfc6265
     http://tools.ietf.org/html/rfc2109
@@ -31,8 +28,21 @@ _cookie_params = set((
     'secure', 'httponly', 'version',
 ))
 
+ESCAPE = re.compile(r"([\"\\])")
 
-# TODO: Disallow LHS-only Cookie values
+
+class CookieAttrs(multidict.ImmutableMultiDict):
+    @staticmethod
+    def _kconv(key):
+        return key.lower()
+
+    @staticmethod
+    def _reduce_values(values):
+        # See the StickyCookieTest for a weird cookie that only makes sense
+        # if we take the last part.
+        return values[-1]
+
+SetCookie = collections.namedtuple("SetCookie", ["value", "attrs"])
 
 
 def _read_until(s, start, term):
@@ -47,13 +57,6 @@ def _read_until(s, start, term):
     return s[start:i + 1], i + 1
 
 
-def _read_token(s, start):
-    """
-        Read a token - the LHS of a token/value pair in a cookie.
-    """
-    return _read_until(s, start, ";=")
-
-
 def _read_quoted_string(s, start):
     """
         start: offset to the first quote of the string to be read
@@ -81,12 +84,16 @@ def _read_quoted_string(s, start):
     return "".join(ret), i + 1
 
 
+def _read_key(s, start, delims=";="):
+    """
+        Read a key - the LHS of a token/value pair in a cookie.
+    """
+    return _read_until(s, start, delims)
+
+
 def _read_value(s, start, delims):
     """
         Reads a value - the RHS of a token/value pair in a cookie.
-
-        special: If the value is special, commas are premitted. Else comma
-        terminates. This helps us support old and new style values.
     """
     if start >= len(s):
         return "", start
@@ -96,27 +103,82 @@ def _read_value(s, start, delims):
         return _read_until(s, start, delims)
 
 
-def _read_pairs(s, off=0):
+def _read_cookie_pairs(s, off=0):
     """
-        Read pairs of lhs=rhs values.
+        Read pairs of lhs=rhs values from Cookie headers.
 
         off: start offset
-        specials: a lower-cased list of keys that may contain commas
     """
-    vals = []
+    pairs = []
+
     while True:
-        lhs, off = _read_token(s, off)
+        lhs, off = _read_key(s, off)
         lhs = lhs.lstrip()
+
         if lhs:
             rhs = None
-            if off < len(s):
+            if off < len(s) and s[off] == "=":
-                if s[off] == "=":
                 rhs, off = _read_value(s, off + 1, ";")
-            vals.append([lhs, rhs])
+
+            pairs.append([lhs, rhs])
+
         off += 1
+
         if not off < len(s):
             break
-    return vals, off
+
+    return pairs, off
+
+
+def _read_set_cookie_pairs(s, off=0):
+    """
+        Read pairs of lhs=rhs values from SetCookie headers while handling multiple cookies.
+
+        off: start offset
+        specials: attributes that are treated specially
+    """
+    cookies = []
+    pairs = []
+
+    while True:
+        lhs, off = _read_key(s, off, ";=,")
+        lhs = lhs.lstrip()
+
+        if lhs:
+            rhs = None
+            if off < len(s) and s[off] == "=":
+                rhs, off = _read_value(s, off + 1, ";,")
+
+                # Special handliing of attributes
+                if lhs.lower() == "expires":
+                    # 'expires' values can contain commas in them so they need to
+                    # be handled separately.
+
+                    # We actually bank on the fact that the expires value WILL
+                    # contain a comma. Things will fail, if they don't.
+
+                    # '3' is just a heuristic we use to determine whether we've
+                    # only read a part of the expires value and we should read more.
+                    if len(rhs) <= 3:
+                        trail, off = _read_value(s, off + 1, ";,")
+                        rhs = rhs + "," + trail
+
+            pairs.append([lhs, rhs])
+
+            # comma marks the beginning of a new cookie
+            if off < len(s) and s[off] == ",":
+                cookies.append(pairs)
+                pairs = []
+
+        off += 1
+
+        if not off < len(s):
+            break
+
+    if pairs or not cookies:
+        cookies.append(pairs)
+
+    return cookies, off
 
 
 def _has_special(s):
@@ -129,15 +191,12 @@ def _has_special(s):
     return False
 
 
-ESCAPE = re.compile(r"([\"\\])")
+def _format_pairs(pairs, specials=(), sep="; "):
-
-
-def _format_pairs(lst, specials=(), sep="; "):
     """
         specials: A lower-cased list of keys that will not be quoted.
     """
     vals = []
-    for k, v in lst:
+    for k, v in pairs:
         if v is None:
             vals.append(k)
         else:
@@ -155,64 +214,15 @@ def _format_set_cookie_pairs(lst):
     )
 
 
-def _parse_set_cookie_pairs(s):
+def parse_cookie_header(line):
     """
-        For Set-Cookie, we support multiple cookies as described in RFC2109.
+        Parse a Cookie header value.
-        This function therefore returns a list of lists.
+        Returns a list of (lhs, rhs) tuples.
     """
-    pairs, off_ = _read_pairs(s)
+    pairs, off_ = _read_cookie_pairs(line)
     return pairs
 
 
-def parse_set_cookie_headers(headers):
-    ret = []
-    for header in headers:
-        v = parse_set_cookie_header(header)
-        if v:
-            name, value, attrs = v
-            ret.append((name, SetCookie(value, attrs)))
-    return ret
-
-
-class CookieAttrs(multidict.ImmutableMultiDict):
-    @staticmethod
-    def _kconv(key):
-        return key.lower()
-
-    @staticmethod
-    def _reduce_values(values):
-        # See the StickyCookieTest for a weird cookie that only makes sense
-        # if we take the last part.
-        return values[-1]
-
-
-SetCookie = collections.namedtuple("SetCookie", ["value", "attrs"])
-
-
-def parse_set_cookie_header(line):
-    """
-        Parse a Set-Cookie header value
-
-        Returns a (name, value, attrs) tuple, or None, where attrs is an
-        CookieAttrs dict of attributes. No attempt is made to parse attribute
-        values - they are treated purely as strings.
-    """
-    pairs = _parse_set_cookie_pairs(line)
-    if pairs:
-        return pairs[0][0], pairs[0][1], CookieAttrs(tuple(x) for x in pairs[1:])
-
-
-def format_set_cookie_header(name, value, attrs):
-    """
-        Formats a Set-Cookie header value.
-    """
-    pairs = [(name, value)]
-    pairs.extend(
-        attrs.fields if hasattr(attrs, "fields") else attrs
-    )
-    return _format_set_cookie_pairs(pairs)
-
-
 def parse_cookie_headers(cookie_headers):
     cookie_list = []
     for header in cookie_headers:
@@ -220,15 +230,6 @@ def parse_cookie_headers(cookie_headers):
     return cookie_list
 
 
-def parse_cookie_header(line):
-    """
-        Parse a Cookie header value.
-        Returns a list of (lhs, rhs) tuples.
-    """
-    pairs, off_ = _read_pairs(line)
-    return pairs
-
-
 def format_cookie_header(lst):
     """
         Formats a Cookie header value.
@@ -236,6 +237,57 @@ def format_cookie_header(lst):
     return _format_pairs(lst)
 
 
+def parse_set_cookie_header(line):
+    """
+        Parse a Set-Cookie header value
+
+        Returns a list of (name, value, attrs) tuple for each cokie, or None.
+        Where attrs is a CookieAttrs dict of attributes. No attempt is made
+        to parse attribute values - they are treated purely as strings.
+    """
+    cookie_pairs, off = _read_set_cookie_pairs(line)
+
+    cookies = [
+        (pairs[0][0], pairs[0][1], CookieAttrs(tuple(x) for x in pairs[1:]))
+        for pairs in cookie_pairs if pairs
+    ]
+
+    if cookies:
+        return cookies
+    else:
+        return None
+
+
+def parse_set_cookie_headers(headers):
+    rv = []
+    for header in headers:
+        cookies = parse_set_cookie_header(header)
+        if cookies:
+            for name, value, attrs in cookies:
+                rv.append((name, SetCookie(value, attrs)))
+    return rv
+
+
+def format_set_cookie_header(set_cookies):
+    """
+        Formats a Set-Cookie header value.
+    """
+
+    rv = []
+
+    for set_cookie in set_cookies:
+        name, value, attrs = set_cookie
+
+        pairs = [(name, value)]
+        pairs.extend(
+            attrs.fields if hasattr(attrs, "fields") else attrs
+        )
+
+        rv.append(_format_set_cookie_pairs(pairs))
+
+    return ", ".join(rv)
+
+
 def refresh_set_cookie_header(c, delta):
     """
     Args:
@@ -245,7 +297,7 @@ def refresh_set_cookie_header(c, delta):
         A refreshed Set-Cookie string
     """
 
-    name, value, attrs = parse_set_cookie_header(c)
+    name, value, attrs = parse_set_cookie_header(c)[0]
     if not name or not value:
         raise ValueError("Invalid Cookie")
 
@@ -263,10 +315,10 @@ def refresh_set_cookie_header(c, delta):
             # For now, we just ignore this.
             attrs = attrs.with_delitem("expires")
 
-    ret = format_set_cookie_header(name, value, attrs)
+    rv = format_set_cookie_header([(name, value, attrs)])
-    if not ret:
+    if not rv:
         raise ValueError("Invalid Cookie")
-    return ret
+    return rv
 
 
 def get_expiration_ts(cookie_attrs):

netlib/http/response.py

@@ -155,7 +155,7 @@ class Response(message.Message):
     def _set_cookies(self, value):
         cookie_headers = []
         for k, v in value:
-            header = cookies.format_set_cookie_header(k, v[0], v[1])
+            header = cookies.format_set_cookie_header([(k, v[0], v[1])])
             cookie_headers.append(header)
         self.headers.set_all("set-cookie", cookie_headers)
 

test/mitmproxy/builtins/test_stickycookie.py

@@ -92,7 +92,6 @@ class TestStickyCookie(mastertest.MasterTest):
             "foo/bar=hello",
             "foo:bar=world",
             "foo@bar=fizz",
-            "foo,bar=buzz",
         ]
         for c in cs:
             f.response.headers["Set-Cookie"] = c

test/netlib/http/test_cookies.py

@@ -5,71 +5,7 @@ from netlib.tutils import raises
 
 import mock
 
-
+cookie_pairs = [
-def test_read_token():
-    tokens = [
-        [("foo", 0), ("foo", 3)],
-        [("foo", 1), ("oo", 3)],
-        [(" foo", 1), ("foo", 4)],
-        [(" foo;", 1), ("foo", 4)],
-        [(" foo=", 1), ("foo", 4)],
-        [(" foo=bar", 1), ("foo", 4)],
-    ]
-    for q, a in tokens:
-        assert cookies._read_token(*q) == a
-
-
-def test_read_quoted_string():
-    tokens = [
-        [('"foo" x', 0), ("foo", 5)],
-        [('"f\oo" x', 0), ("foo", 6)],
-        [(r'"f\\o" x', 0), (r"f\o", 6)],
-        [(r'"f\\" x', 0), (r"f" + '\\', 5)],
-        [('"fo\\\"" x', 0), ("fo\"", 6)],
-        [('"foo" x', 7), ("", 8)],
-    ]
-    for q, a in tokens:
-        assert cookies._read_quoted_string(*q) == a
-
-
-def test_read_pairs():
-    vals = [
-        [
-            "one",
-            [["one", None]]
-        ],
-        [
-            "one=two",
-            [["one", "two"]]
-        ],
-        [
-            "one=",
-            [["one", ""]]
-        ],
-        [
-            'one="two"',
-            [["one", "two"]]
-        ],
-        [
-            'one="two"; three=four',
-            [["one", "two"], ["three", "four"]]
-        ],
-        [
-            'one="two"; three=four; five',
-            [["one", "two"], ["three", "four"], ["five", None]]
-        ],
-        [
-            'one="\\"two"; three=four',
-            [["one", '"two'], ["three", "four"]]
-        ],
-    ]
-    for s, lst in vals:
-        ret, off = cookies._read_pairs(s)
-        assert ret == lst
-
-
-def test_pairs_roundtrips():
-    pairs = [
     [
         "",
         []
@@ -111,87 +47,151 @@ def test_pairs_roundtrips():
         ]
     ]
 ]
-    for s, lst in pairs:
+
-        ret, off = cookies._read_pairs(s)
+
-        assert ret == lst
+def test_read_key():
-        s2 = cookies._format_pairs(lst)
+    tokens = [
-        ret, off = cookies._read_pairs(s2)
+        [("foo", 0), ("foo", 3)],
+        [("foo", 1), ("oo", 3)],
+        [(" foo", 0), (" foo", 4)],
+        [(" foo", 1), ("foo", 4)],
+        [(" foo;", 1), ("foo", 4)],
+        [(" foo=", 1), ("foo", 4)],
+        [(" foo=bar", 1), ("foo", 4)],
+    ]
+    for q, a in tokens:
+        assert cookies._read_key(*q) == a
+
+
+def test_read_quoted_string():
+    tokens = [
+        [('"foo" x', 0), ("foo", 5)],
+        [('"f\oo" x', 0), ("foo", 6)],
+        [(r'"f\\o" x', 0), (r"f\o", 6)],
+        [(r'"f\\" x', 0), (r"f" + '\\', 5)],
+        [('"fo\\\"" x', 0), ("fo\"", 6)],
+        [('"foo" x', 7), ("", 8)],
+    ]
+    for q, a in tokens:
+        assert cookies._read_quoted_string(*q) == a
+
+
+def test_read_cookie_pairs():
+    vals = [
+        [
+            "one",
+            [["one", None]]
+        ],
+        [
+            "one=two",
+            [["one", "two"]]
+        ],
+        [
+            "one=",
+            [["one", ""]]
+        ],
+        [
+            'one="two"',
+            [["one", "two"]]
+        ],
+        [
+            'one="two"; three=four',
+            [["one", "two"], ["three", "four"]]
+        ],
+        [
+            'one="two"; three=four; five',
+            [["one", "two"], ["three", "four"], ["five", None]]
+        ],
+        [
+            'one="\\"two"; three=four',
+            [["one", '"two'], ["three", "four"]]
+        ],
+    ]
+    for s, lst in vals:
+        ret, off = cookies._read_cookie_pairs(s)
         assert ret == lst
 
 
+def test_pairs_roundtrips():
+    for s, expected in cookie_pairs:
+        ret, off = cookies._read_cookie_pairs(s)
+        assert ret == expected
+
+        s2 = cookies._format_pairs(expected)
+        ret, off = cookies._read_cookie_pairs(s2)
+        assert ret == expected
+
+
 def test_cookie_roundtrips():
-    pairs = [
+    for s, expected in cookie_pairs:
-        [
-            "one=uno",
-            [["one", "uno"]]
-        ],
-        [
-            "one=uno; two=due",
-            [["one", "uno"], ["two", "due"]]
-        ],
-    ]
-    for s, lst in pairs:
         ret = cookies.parse_cookie_header(s)
-        assert ret == lst
+        assert ret == expected
-        s2 = cookies.format_cookie_header(ret)
+
+        s2 = cookies.format_cookie_header(expected)
         ret = cookies.parse_cookie_header(s2)
-        assert ret == lst
+        assert ret == expected
 
 
 def test_parse_set_cookie_pairs():
     pairs = [
         [
             "one=uno",
-            [
+            [[
                 ["one", "uno"]
-            ]
+            ]]
         ],
         [
             "one=un\x20",
-            [
+            [[
                 ["one", "un\x20"]
-            ]
+            ]]
         ],
         [
             "one=uno; foo",
-            [
+            [[
                 ["one", "uno"],
                 ["foo", None]
-            ]
+            ]]
         ],
         [
             "mun=1.390.f60; "
             "expires=sun, 11-oct-2015 12:38:31 gmt; path=/; "
             "domain=b.aol.com",
-            [
+            [[
                 ["mun", "1.390.f60"],
                 ["expires", "sun, 11-oct-2015 12:38:31 gmt"],
                 ["path", "/"],
                 ["domain", "b.aol.com"]
-            ]
+            ]]
         ],
         [
             r'rpb=190%3d1%2616726%3d1%2634832%3d1%2634874%3d1; '
             'domain=.rubiconproject.com; '
             'expires=mon, 11-may-2015 21:54:57 gmt; '
             'path=/',
-            [
+            [[
                 ['rpb', r'190%3d1%2616726%3d1%2634832%3d1%2634874%3d1'],
                 ['domain', '.rubiconproject.com'],
                 ['expires', 'mon, 11-may-2015 21:54:57 gmt'],
                 ['path', '/']
-            ]
+            ]]
         ],
     ]
-    for s, lst in pairs:
+    for s, expected in pairs:
-        ret = cookies._parse_set_cookie_pairs(s)
+        ret, off = cookies._read_set_cookie_pairs(s)
-        assert ret == lst
+        assert ret == expected
-        s2 = cookies._format_set_cookie_pairs(ret)
+
-        ret2 = cookies._parse_set_cookie_pairs(s2)
+        s2 = cookies._format_set_cookie_pairs(expected[0])
-        assert ret2 == lst
+        ret2, off = cookies._read_set_cookie_pairs(s2)
+        assert ret2 == expected
 
 
 def test_parse_set_cookie_header():
+    def set_cookie_equal(obs, exp):
+        assert obs[0] == exp[0]
+        assert obs[1] == exp[1]
+        assert obs[2].items(multi=True) == exp[2]
+
     vals = [
         [
             "", None
@@ -201,28 +201,61 @@ def test_parse_set_cookie_header():
         ],
         [
             "one=uno",
+            [
                 ("one", "uno", ())
+            ]
         ],
         [
             "one=uno; foo=bar",
+            [
                 ("one", "uno", (("foo", "bar"),))
+            ]
         ],
         [
             "one=uno; foo=bar; foo=baz",
+            [
                 ("one", "uno", (("foo", "bar"), ("foo", "baz")))
+            ]
+        ],
+        # Comma Separated Variant of Set-Cookie Headers
+        [
+            "foo=bar, doo=dar",
+            [
+                ("foo", "bar", ()),
+                ("doo", "dar", ()),
+            ]
+        ],
+        [
+            "foo=bar; path=/, doo=dar; roo=rar; zoo=zar",
+            [
+                ("foo", "bar", (("path", "/"),)),
+                ("doo", "dar", (("roo", "rar"), ("zoo", "zar"))),
+            ]
+        ],
+        [
+            "foo=bar; expires=Mon, 24 Aug 2037",
+            [
+                ("foo", "bar", (("expires", "Mon, 24 Aug 2037"),)),
+            ]
+        ],
+        [
+            "foo=bar; expires=Mon, 24 Aug 2037 00:00:00 GMT, doo=dar",
+            [
+                ("foo", "bar", (("expires", "Mon, 24 Aug 2037 00:00:00 GMT"),)),
+                ("doo", "dar", ()),
+            ]
         ],
     ]
     for s, expected in vals:
         ret = cookies.parse_set_cookie_header(s)
         if expected:
-            assert ret[0] == expected[0]
+            for i in range(len(expected)):
-            assert ret[1] == expected[1]
+                set_cookie_equal(ret[i], expected[i])
-            assert ret[2].items(multi=True) == expected[2]
+
-            s2 = cookies.format_set_cookie_header(*ret)
+            s2 = cookies.format_set_cookie_header(ret)
             ret2 = cookies.parse_set_cookie_header(s2)
-            assert ret2[0] == expected[0]
+            for i in range(len(expected)):
-            assert ret2[1] == expected[1]
+                set_cookie_equal(ret2[i], expected[i])
-            assert ret2[2].items(multi=True) == expected[2]
         else:
             assert ret is None