This commit is contained in:
Aldo Cortesi 2016-09-28 12:44:40 +10:00
commit b21f076cc8
4 changed files with 276 additions and 192 deletions

View File

@ -14,12 +14,9 @@ information. Duplicate cookies are preserved in parsing, and can be set in
formatting. We do attempt to escape and quote values where needed, but will not
reject data that violate the specs.
Parsing accepts the formats in RFC6265 and partially RFC2109 and RFC2965. We do
not parse the comma-separated variant of Set-Cookie that allows multiple
cookies to be set in a single header. Technically this should be feasible, but
it turns out that violations of RFC6265 that makes the parsing problem
indeterminate are much more common than genuine occurences of the multi-cookie
variants. Serialization follows RFC6265.
Parsing accepts the formats in RFC6265 and partially RFC2109 and RFC2965. We
also parse the comma-separated variant of Set-Cookie that allows multiple
cookies to be set in a single header. Serialization follows RFC6265.
http://tools.ietf.org/html/rfc6265
http://tools.ietf.org/html/rfc2109
@ -31,8 +28,21 @@ _cookie_params = set((
'secure', 'httponly', 'version',
))
ESCAPE = re.compile(r"([\"\\])")
# TODO: Disallow LHS-only Cookie values
class CookieAttrs(multidict.ImmutableMultiDict):
@staticmethod
def _kconv(key):
return key.lower()
@staticmethod
def _reduce_values(values):
# See the StickyCookieTest for a weird cookie that only makes sense
# if we take the last part.
return values[-1]
SetCookie = collections.namedtuple("SetCookie", ["value", "attrs"])
def _read_until(s, start, term):
@ -47,13 +57,6 @@ def _read_until(s, start, term):
return s[start:i + 1], i + 1
def _read_token(s, start):
"""
Read a token - the LHS of a token/value pair in a cookie.
"""
return _read_until(s, start, ";=")
def _read_quoted_string(s, start):
"""
start: offset to the first quote of the string to be read
@ -81,12 +84,16 @@ def _read_quoted_string(s, start):
return "".join(ret), i + 1
def _read_key(s, start, delims=";="):
"""
Read a key - the LHS of a token/value pair in a cookie.
"""
return _read_until(s, start, delims)
def _read_value(s, start, delims):
"""
Reads a value - the RHS of a token/value pair in a cookie.
special: If the value is special, commas are premitted. Else comma
terminates. This helps us support old and new style values.
"""
if start >= len(s):
return "", start
@ -96,27 +103,82 @@ def _read_value(s, start, delims):
return _read_until(s, start, delims)
def _read_pairs(s, off=0):
def _read_cookie_pairs(s, off=0):
"""
Read pairs of lhs=rhs values.
Read pairs of lhs=rhs values from Cookie headers.
off: start offset
specials: a lower-cased list of keys that may contain commas
"""
vals = []
pairs = []
while True:
lhs, off = _read_token(s, off)
lhs, off = _read_key(s, off)
lhs = lhs.lstrip()
if lhs:
rhs = None
if off < len(s):
if s[off] == "=":
if off < len(s) and s[off] == "=":
rhs, off = _read_value(s, off + 1, ";")
vals.append([lhs, rhs])
pairs.append([lhs, rhs])
off += 1
if not off < len(s):
break
return vals, off
return pairs, off
def _read_set_cookie_pairs(s, off=0):
"""
Read pairs of lhs=rhs values from SetCookie headers while handling multiple cookies.
off: start offset
specials: attributes that are treated specially
"""
cookies = []
pairs = []
while True:
lhs, off = _read_key(s, off, ";=,")
lhs = lhs.lstrip()
if lhs:
rhs = None
if off < len(s) and s[off] == "=":
rhs, off = _read_value(s, off + 1, ";,")
# Special handliing of attributes
if lhs.lower() == "expires":
# 'expires' values can contain commas in them so they need to
# be handled separately.
# We actually bank on the fact that the expires value WILL
# contain a comma. Things will fail, if they don't.
# '3' is just a heuristic we use to determine whether we've
# only read a part of the expires value and we should read more.
if len(rhs) <= 3:
trail, off = _read_value(s, off + 1, ";,")
rhs = rhs + "," + trail
pairs.append([lhs, rhs])
# comma marks the beginning of a new cookie
if off < len(s) and s[off] == ",":
cookies.append(pairs)
pairs = []
off += 1
if not off < len(s):
break
if pairs or not cookies:
cookies.append(pairs)
return cookies, off
def _has_special(s):
@ -129,15 +191,12 @@ def _has_special(s):
return False
ESCAPE = re.compile(r"([\"\\])")
def _format_pairs(lst, specials=(), sep="; "):
def _format_pairs(pairs, specials=(), sep="; "):
"""
specials: A lower-cased list of keys that will not be quoted.
"""
vals = []
for k, v in lst:
for k, v in pairs:
if v is None:
vals.append(k)
else:
@ -155,64 +214,15 @@ def _format_set_cookie_pairs(lst):
)
def _parse_set_cookie_pairs(s):
def parse_cookie_header(line):
"""
For Set-Cookie, we support multiple cookies as described in RFC2109.
This function therefore returns a list of lists.
Parse a Cookie header value.
Returns a list of (lhs, rhs) tuples.
"""
pairs, off_ = _read_pairs(s)
pairs, off_ = _read_cookie_pairs(line)
return pairs
def parse_set_cookie_headers(headers):
ret = []
for header in headers:
v = parse_set_cookie_header(header)
if v:
name, value, attrs = v
ret.append((name, SetCookie(value, attrs)))
return ret
class CookieAttrs(multidict.ImmutableMultiDict):
@staticmethod
def _kconv(key):
return key.lower()
@staticmethod
def _reduce_values(values):
# See the StickyCookieTest for a weird cookie that only makes sense
# if we take the last part.
return values[-1]
SetCookie = collections.namedtuple("SetCookie", ["value", "attrs"])
def parse_set_cookie_header(line):
"""
Parse a Set-Cookie header value
Returns a (name, value, attrs) tuple, or None, where attrs is an
CookieAttrs dict of attributes. No attempt is made to parse attribute
values - they are treated purely as strings.
"""
pairs = _parse_set_cookie_pairs(line)
if pairs:
return pairs[0][0], pairs[0][1], CookieAttrs(tuple(x) for x in pairs[1:])
def format_set_cookie_header(name, value, attrs):
"""
Formats a Set-Cookie header value.
"""
pairs = [(name, value)]
pairs.extend(
attrs.fields if hasattr(attrs, "fields") else attrs
)
return _format_set_cookie_pairs(pairs)
def parse_cookie_headers(cookie_headers):
cookie_list = []
for header in cookie_headers:
@ -220,15 +230,6 @@ def parse_cookie_headers(cookie_headers):
return cookie_list
def parse_cookie_header(line):
"""
Parse a Cookie header value.
Returns a list of (lhs, rhs) tuples.
"""
pairs, off_ = _read_pairs(line)
return pairs
def format_cookie_header(lst):
"""
Formats a Cookie header value.
@ -236,6 +237,57 @@ def format_cookie_header(lst):
return _format_pairs(lst)
def parse_set_cookie_header(line):
"""
Parse a Set-Cookie header value
Returns a list of (name, value, attrs) tuple for each cokie, or None.
Where attrs is a CookieAttrs dict of attributes. No attempt is made
to parse attribute values - they are treated purely as strings.
"""
cookie_pairs, off = _read_set_cookie_pairs(line)
cookies = [
(pairs[0][0], pairs[0][1], CookieAttrs(tuple(x) for x in pairs[1:]))
for pairs in cookie_pairs if pairs
]
if cookies:
return cookies
else:
return None
def parse_set_cookie_headers(headers):
rv = []
for header in headers:
cookies = parse_set_cookie_header(header)
if cookies:
for name, value, attrs in cookies:
rv.append((name, SetCookie(value, attrs)))
return rv
def format_set_cookie_header(set_cookies):
"""
Formats a Set-Cookie header value.
"""
rv = []
for set_cookie in set_cookies:
name, value, attrs = set_cookie
pairs = [(name, value)]
pairs.extend(
attrs.fields if hasattr(attrs, "fields") else attrs
)
rv.append(_format_set_cookie_pairs(pairs))
return ", ".join(rv)
def refresh_set_cookie_header(c, delta):
"""
Args:
@ -245,7 +297,7 @@ def refresh_set_cookie_header(c, delta):
A refreshed Set-Cookie string
"""
name, value, attrs = parse_set_cookie_header(c)
name, value, attrs = parse_set_cookie_header(c)[0]
if not name or not value:
raise ValueError("Invalid Cookie")
@ -263,10 +315,10 @@ def refresh_set_cookie_header(c, delta):
# For now, we just ignore this.
attrs = attrs.with_delitem("expires")
ret = format_set_cookie_header(name, value, attrs)
if not ret:
rv = format_set_cookie_header([(name, value, attrs)])
if not rv:
raise ValueError("Invalid Cookie")
return ret
return rv
def get_expiration_ts(cookie_attrs):

View File

@ -155,7 +155,7 @@ class Response(message.Message):
def _set_cookies(self, value):
cookie_headers = []
for k, v in value:
header = cookies.format_set_cookie_header(k, v[0], v[1])
header = cookies.format_set_cookie_header([(k, v[0], v[1])])
cookie_headers.append(header)
self.headers.set_all("set-cookie", cookie_headers)

View File

@ -92,7 +92,6 @@ class TestStickyCookie(mastertest.MasterTest):
"foo/bar=hello",
"foo:bar=world",
"foo@bar=fizz",
"foo,bar=buzz",
]
for c in cs:
f.response.headers["Set-Cookie"] = c

View File

@ -5,71 +5,7 @@ from netlib.tutils import raises
import mock
def test_read_token():
tokens = [
[("foo", 0), ("foo", 3)],
[("foo", 1), ("oo", 3)],
[(" foo", 1), ("foo", 4)],
[(" foo;", 1), ("foo", 4)],
[(" foo=", 1), ("foo", 4)],
[(" foo=bar", 1), ("foo", 4)],
]
for q, a in tokens:
assert cookies._read_token(*q) == a
def test_read_quoted_string():
tokens = [
[('"foo" x', 0), ("foo", 5)],
[('"f\oo" x', 0), ("foo", 6)],
[(r'"f\\o" x', 0), (r"f\o", 6)],
[(r'"f\\" x', 0), (r"f" + '\\', 5)],
[('"fo\\\"" x', 0), ("fo\"", 6)],
[('"foo" x', 7), ("", 8)],
]
for q, a in tokens:
assert cookies._read_quoted_string(*q) == a
def test_read_pairs():
vals = [
[
"one",
[["one", None]]
],
[
"one=two",
[["one", "two"]]
],
[
"one=",
[["one", ""]]
],
[
'one="two"',
[["one", "two"]]
],
[
'one="two"; three=four',
[["one", "two"], ["three", "four"]]
],
[
'one="two"; three=four; five',
[["one", "two"], ["three", "four"], ["five", None]]
],
[
'one="\\"two"; three=four',
[["one", '"two'], ["three", "four"]]
],
]
for s, lst in vals:
ret, off = cookies._read_pairs(s)
assert ret == lst
def test_pairs_roundtrips():
pairs = [
cookie_pairs = [
[
"",
[]
@ -110,88 +46,152 @@ def test_pairs_roundtrips():
["_rcc2", "53VdltWl+Ov6ordflA=="]
]
]
]
def test_read_key():
tokens = [
[("foo", 0), ("foo", 3)],
[("foo", 1), ("oo", 3)],
[(" foo", 0), (" foo", 4)],
[(" foo", 1), ("foo", 4)],
[(" foo;", 1), ("foo", 4)],
[(" foo=", 1), ("foo", 4)],
[(" foo=bar", 1), ("foo", 4)],
]
for s, lst in pairs:
ret, off = cookies._read_pairs(s)
assert ret == lst
s2 = cookies._format_pairs(lst)
ret, off = cookies._read_pairs(s2)
for q, a in tokens:
assert cookies._read_key(*q) == a
def test_read_quoted_string():
tokens = [
[('"foo" x', 0), ("foo", 5)],
[('"f\oo" x', 0), ("foo", 6)],
[(r'"f\\o" x', 0), (r"f\o", 6)],
[(r'"f\\" x', 0), (r"f" + '\\', 5)],
[('"fo\\\"" x', 0), ("fo\"", 6)],
[('"foo" x', 7), ("", 8)],
]
for q, a in tokens:
assert cookies._read_quoted_string(*q) == a
def test_read_cookie_pairs():
vals = [
[
"one",
[["one", None]]
],
[
"one=two",
[["one", "two"]]
],
[
"one=",
[["one", ""]]
],
[
'one="two"',
[["one", "two"]]
],
[
'one="two"; three=four',
[["one", "two"], ["three", "four"]]
],
[
'one="two"; three=four; five',
[["one", "two"], ["three", "four"], ["five", None]]
],
[
'one="\\"two"; three=four',
[["one", '"two'], ["three", "four"]]
],
]
for s, lst in vals:
ret, off = cookies._read_cookie_pairs(s)
assert ret == lst
def test_pairs_roundtrips():
for s, expected in cookie_pairs:
ret, off = cookies._read_cookie_pairs(s)
assert ret == expected
s2 = cookies._format_pairs(expected)
ret, off = cookies._read_cookie_pairs(s2)
assert ret == expected
def test_cookie_roundtrips():
pairs = [
[
"one=uno",
[["one", "uno"]]
],
[
"one=uno; two=due",
[["one", "uno"], ["two", "due"]]
],
]
for s, lst in pairs:
for s, expected in cookie_pairs:
ret = cookies.parse_cookie_header(s)
assert ret == lst
s2 = cookies.format_cookie_header(ret)
assert ret == expected
s2 = cookies.format_cookie_header(expected)
ret = cookies.parse_cookie_header(s2)
assert ret == lst
assert ret == expected
def test_parse_set_cookie_pairs():
pairs = [
[
"one=uno",
[
[[
["one", "uno"]
]
]]
],
[
"one=un\x20",
[
[[
["one", "un\x20"]
]
]]
],
[
"one=uno; foo",
[
[[
["one", "uno"],
["foo", None]
]
]]
],
[
"mun=1.390.f60; "
"expires=sun, 11-oct-2015 12:38:31 gmt; path=/; "
"domain=b.aol.com",
[
[[
["mun", "1.390.f60"],
["expires", "sun, 11-oct-2015 12:38:31 gmt"],
["path", "/"],
["domain", "b.aol.com"]
]
]]
],
[
r'rpb=190%3d1%2616726%3d1%2634832%3d1%2634874%3d1; '
'domain=.rubiconproject.com; '
'expires=mon, 11-may-2015 21:54:57 gmt; '
'path=/',
[
[[
['rpb', r'190%3d1%2616726%3d1%2634832%3d1%2634874%3d1'],
['domain', '.rubiconproject.com'],
['expires', 'mon, 11-may-2015 21:54:57 gmt'],
['path', '/']
]
]]
],
]
for s, lst in pairs:
ret = cookies._parse_set_cookie_pairs(s)
assert ret == lst
s2 = cookies._format_set_cookie_pairs(ret)
ret2 = cookies._parse_set_cookie_pairs(s2)
assert ret2 == lst
for s, expected in pairs:
ret, off = cookies._read_set_cookie_pairs(s)
assert ret == expected
s2 = cookies._format_set_cookie_pairs(expected[0])
ret2, off = cookies._read_set_cookie_pairs(s2)
assert ret2 == expected
def test_parse_set_cookie_header():
def set_cookie_equal(obs, exp):
assert obs[0] == exp[0]
assert obs[1] == exp[1]
assert obs[2].items(multi=True) == exp[2]
vals = [
[
"", None
@ -201,28 +201,61 @@ def test_parse_set_cookie_header():
],
[
"one=uno",
[
("one", "uno", ())
]
],
[
"one=uno; foo=bar",
[
("one", "uno", (("foo", "bar"),))
]
],
[
"one=uno; foo=bar; foo=baz",
[
("one", "uno", (("foo", "bar"), ("foo", "baz")))
]
],
# Comma Separated Variant of Set-Cookie Headers
[
"foo=bar, doo=dar",
[
("foo", "bar", ()),
("doo", "dar", ()),
]
],
[
"foo=bar; path=/, doo=dar; roo=rar; zoo=zar",
[
("foo", "bar", (("path", "/"),)),
("doo", "dar", (("roo", "rar"), ("zoo", "zar"))),
]
],
[
"foo=bar; expires=Mon, 24 Aug 2037",
[
("foo", "bar", (("expires", "Mon, 24 Aug 2037"),)),
]
],
[
"foo=bar; expires=Mon, 24 Aug 2037 00:00:00 GMT, doo=dar",
[
("foo", "bar", (("expires", "Mon, 24 Aug 2037 00:00:00 GMT"),)),
("doo", "dar", ()),
]
],
]
for s, expected in vals:
ret = cookies.parse_set_cookie_header(s)
if expected:
assert ret[0] == expected[0]
assert ret[1] == expected[1]
assert ret[2].items(multi=True) == expected[2]
s2 = cookies.format_set_cookie_header(*ret)
for i in range(len(expected)):
set_cookie_equal(ret[i], expected[i])
s2 = cookies.format_set_cookie_header(ret)
ret2 = cookies.parse_set_cookie_header(s2)
assert ret2[0] == expected[0]
assert ret2[1] == expected[1]
assert ret2[2].items(multi=True) == expected[2]
for i in range(len(expected)):
set_cookie_equal(ret2[i], expected[i])
else:
assert ret is None