Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-26 18:18:25 +00:00
Code Issues Projects Releases Wiki Activity

A clearer implementation of MultiDictView

Browse Source 
This makes MultiDictView work with a simple getter/setter pair, rather than
using attributes with implicit leading underscores. Also move MultiDictView
into multidict.py and adds some simple unit tests.
This commit is contained in:
Aldo Cortesi 2016-05-21 11:37:36 +12:00
parent 96d8ec1ee3
commit a5c4cd0340
6 changed files with 119 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
netlib/http/__init__.py
View File

@ -2,13 +2,13 @@ from __future__ import absolute_import, print_function, division
from .request import Request
from .response import Response
from .headers import Headers
from .message import MultiDictView, decoded
from .message import decoded
from . import http1, http2, status_codes
__all__ = [
"Request",
"Response",
"Headers",
"MultiDictView", "decoded",
"decoded",
"http1", "http2", "status_codes",
]

69
netlib/http/message.py
View File

@ -236,72 +236,3 @@ class decoded(object):
def __exit__(self, type, value, tb):
if self.ce:
self.message.encode(self.ce)
class MultiDictView(MultiDict):
"""
Some parts in HTTP (Cookies, URL query strings, ...) require a specific data structure: A MultiDict.
It behaves mostly like an ordered dict but it can have several values for the same key.
The MultiDictView provides a MultiDict *view* on an :py:class:`Request` or :py:class:`Response`.
That is, it represents a part of the request as a MultiDict, but doesn't contain state/data themselves.
For example, ``request.cookies`` provides a view on the ``Cookie: ...`` header.
Any change to ``request.cookies`` will also modify the ``Cookie`` header.
Any change to the ``Cookie`` header will also modify ``request.cookies``.
Example:
.. code-block:: python
# Cookies are represented as a MultiDict.
>>> request.cookies
MultiDictView[("name", "value"), ("a", "false"), ("a", "42")]
# MultiDicts mostly behave like a normal dict.
>>> request.cookies["name"]
"value"
# If there is more than one value, only the first value is returned.
>>> request.cookies["a"]
"false"
# `.get_all(key)` returns a list of all values.
>>> request.cookies.get_all("a")
["false", "42"]
# Changes to the headers are immediately reflected in the cookies.
>>> request.cookies
MultiDictView[("name", "value"), ...]
>>> del request.headers["Cookie"]
>>> request.cookies
MultiDictView[] # empty now
"""
def __init__(self, attr, message):
if False: # pragma: no cover
# We do not want to call the parent constructor here as that
# would cause an unnecessary parse/unparse pass.
# This is here to silence linters. Message
super(MultiDictView, self).__init__(None)
self._attr = attr
self._message = message # type: Message
@staticmethod
def _kconv(key):
# All request-attributes are case-sensitive.
return key
@staticmethod
def _reduce_values(values):
# We just return the first element if
# multiple elements exist with the same key.
return values[0]
@property
def fields(self):
return getattr(self._message, "_" + self._attr)
@fields.setter
def fields(self, value):
setattr(self._message, self._attr, value)

59
netlib/http/request.py
View File

@ -10,8 +10,9 @@ from netlib import utils
from netlib.http import cookies
from netlib.odict import ODict
from .. import encoding
from ..multidict import MultiDictView
from .headers import Headers
from .message import Message, _native, _always_bytes, MessageData, MultiDictView
from .message import Message, _native, _always_bytes, MessageData
# This regex extracts & splits the host header into host and port.
# Handles the edge case of IPv6 addresses containing colons.
@ -228,20 +229,25 @@ class Request(Message):
"""
The request query string as an :py:class:`MultiDictView` object.
"""
return MultiDictView("query", self)
return MultiDictView(
self._get_query,
self._set_query
)
@property
def _query(self):
def _get_query(self):
_, _, _, _, query, _ = urllib.parse.urlparse(self.url)
return tuple(utils.urldecode(query))
@query.setter
def query(self, value):
def _set_query(self, value):
query = utils.urlencode(value)
scheme, netloc, path, params, _, fragment = urllib.parse.urlparse(self.url)
_, _, _, self.path = utils.parse_url(
urllib.parse.urlunparse([scheme, netloc, path, params, query, fragment]))
@query.setter
def query(self, value):
self._set_query(value)
@property
def cookies(self):
# type: () -> MultiDictView
@ -250,16 +256,21 @@ class Request(Message):
An empty :py:class:`MultiDictView` object if the cookie monster ate them all.
"""
return MultiDictView("cookies", self)
return MultiDictView(
self._get_cookies,
self._set_cookies
)
@property
def _cookies(self):
def _get_cookies(self):
h = self.headers.get_all("Cookie")
return tuple(cookies.parse_cookie_headers(h))
def _set_cookies(self, value):
self.headers["cookie"] = cookies.format_cookie_header(value)
@cookies.setter
def cookies(self, value):
self.headers["cookie"] = cookies.format_cookie_header(value)
self._set_cookies(value)
@property
def path_components(self):
@ -322,17 +333,18 @@ class Request(Message):
An empty MultiDictView if the content-type indicates non-form data
or the content could not be parsed.
"""
return MultiDictView("urlencoded_form", self)
return MultiDictView(
self._get_urlencoded_form,
self._set_urlencoded_form
)
@property
def _urlencoded_form(self):
def _get_urlencoded_form(self):
is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower()
if is_valid_content_type:
return tuple(utils.urldecode(self.content))
return ()
@urlencoded_form.setter
def urlencoded_form(self, value):
def _set_urlencoded_form(self, value):
"""
Sets the body to the URL-encoded form data, and adds the appropriate content-type header.
This will overwrite the existing content if there is one.
@ -340,21 +352,30 @@ class Request(Message):
self.headers["content-type"] = "application/x-www-form-urlencoded"
self.content = utils.urlencode(value)
@urlencoded_form.setter
def urlencoded_form(self, value):
self._set_urlencoded_form(value)
@property
def multipart_form(self):
"""
The multipart form data as an :py:class:`MultipartFormDict` object.
None if the content-type indicates non-form data.
"""
return MultiDictView("multipart_form", self)
return MultiDictView(
self._get_multipart_form,
self._set_multipart_form
)
@property
def _multipart_form(self):
def _get_multipart_form(self):
is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower()
if is_valid_content_type:
return utils.multipartdecode(self.headers, self.content)
return ()
def _set_multipart_form(self, value):
raise NotImplementedError()
@multipart_form.setter
def multipart_form(self, value):
raise NotImplementedError()
self._set_multipart_form(value)

20
netlib/http/response.py
View File

@ -5,7 +5,8 @@ import time
from . import cookies
from .headers import Headers
from .message import Message, _native, _always_bytes, MessageData, MultiDictView
from .message import Message, _native, _always_bytes, MessageData
from ..multidict import MultiDictView
from .. import utils
@ -80,21 +81,26 @@ class Response(Message):
Caveats:
Updating the attr
"""
return MultiDictView("cookies", self)
return MultiDictView(
self._get_cookies,
self._set_cookies
)
@property
def _cookies(self):
def _get_cookies(self):
h = self.headers.get_all("set-cookie")
return tuple(cookies.parse_set_cookie_headers(h))
@cookies.setter
def cookies(self, all_cookies):
def _set_cookies(self, value):
cookie_headers = []
for k, v in all_cookies:
for k, v in value:
header = cookies.format_set_cookie_header(k, v[0], v[1])
cookie_headers.append(header)
self.headers.set_all("set-cookie", cookie_headers)
@cookies.setter
def cookies(self, value):
self._set_cookies(value)
def refresh(self, now=None):
"""
This fairly complex and heuristic function refreshes a server

49
netlib/multidict.py
View File

@ -15,13 +15,7 @@ from .utils import Serializable
@six.add_metaclass(ABCMeta)
class MultiDict(MutableMapping, Serializable):
def __init__(self, fields=None):
# it is important for us that .fields is immutable, so that we can easily
# detect changes to it.
self.fields = tuple(fields) if fields else tuple() # type: Tuple[Tuple[bytes, bytes], ...]
class _MultiDict(MutableMapping, Serializable):
def __repr__(self):
fields = tuple(
repr(field)
@ -97,7 +91,7 @@ class MultiDict(MutableMapping, Serializable):
value
for k, value in self.fields
if self._kconv(k) == key
]
]
def set_all(self, key, values):
"""
@ -173,7 +167,7 @@ class MultiDict(MutableMapping, Serializable):
if multi:
return self.fields
else:
return super(MultiDict, self).items()
return super(_MultiDict, self).items()
def to_dict(self):
"""
@ -213,6 +207,12 @@ class MultiDict(MutableMapping, Serializable):
return cls(tuple(x) for x in state)
class MultiDict(_MultiDict):
def __init__(self, fields=None):
super(MultiDict, self).__init__()
self.fields = tuple(fields) if fields else tuple() # type: Tuple[Tuple[bytes, bytes], ...]
@six.add_metaclass(ABCMeta)
class ImmutableMultiDict(MultiDict):
def _immutable(self, *_):
@ -246,3 +246,34 @@ class ImmutableMultiDict(MultiDict):
ret = self.copy()
super(ImmutableMultiDict, ret).insert(index, key, value)
return ret
class MultiDictView(_MultiDict):
"""
The MultiDictView provides the MultiDict interface over calculated data.
The view itself contains no state - data is retrieved from the parent on
request, and stored back to the parent on change.
"""
def __init__(self, getter, setter):
self._getter = getter
self._setter = setter
super(MultiDictView, self).__init__()
@staticmethod
def _kconv(key):
# All request-attributes are case-sensitive.
return key
@staticmethod
def _reduce_values(values):
# We just return the first element if
# multiple elements exist with the same key.
return values[0]
@property
def fields(self):
return self._getter()
@fields.setter
def fields(self, value):
return self._setter(value)

26
test/netlib/test_multidict.py
View File

@ -1,5 +1,5 @@
from netlib import tutils
from netlib.multidict import MultiDict, ImmutableMultiDict
from netlib.multidict import MultiDict, ImmutableMultiDict, MultiDictView
class _TMulti(object):
@ -214,4 +214,26 @@ class TestImmutableMultiDict(object):
def test_with_insert(self):
md = TImmutableMultiDict()
assert md.with_insert(0, "foo", "bar").fields == (("foo", "bar"),)
assert md.fields == ()
class TParent(object):
def __init__(self):
self.vals = tuple()
def setter(self, vals):
self.vals = vals
def getter(self):
return self.vals
class TestMultiDictView(object):
def test_modify(self):
p = TParent()
tv = MultiDictView(p.getter, p.setter)
assert len(tv) == 0
tv["a"] = "b"
assert p.vals == (("a", "b"),)
tv["c"] = "b"
assert p.vals == (("a", "b"), ("c", "b"))
assert tv["a"] == "b"