mitmproxy/netlib/http/message.py

from __future__ import absolute_import, print_function, division

import re
import warnings

import six

from netlib import encoding, strutils, basetypes
from netlib.http import headers

if six.PY2:  # pragma: no cover
    def _native(x):
        return x

    def _always_bytes(x):
        return x
else:
    # While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
    def _native(x):
        return x.decode("utf-8", "surrogateescape")

    def _always_bytes(x):
        return strutils.always_bytes(x, "utf-8", "surrogateescape")


class MessageData(basetypes.Serializable):
    def __eq__(self, other):
        if isinstance(other, MessageData):
            return self.__dict__ == other.__dict__
        return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(frozenset(self.__dict__.items()))

    def set_state(self, state):
        for k, v in state.items():
            if k == "headers":
                v = headers.Headers.from_state(v)
            setattr(self, k, v)

    def get_state(self):
        state = vars(self).copy()
        state["headers"] = state["headers"].get_state()
        return state

    @classmethod
    def from_state(cls, state):
        state["headers"] = headers.Headers.from_state(state["headers"])
        return cls(**state)


class CachedDecode(object):
    __slots__ = ["encoded", "encoding", "strict", "decoded"]

    def __init__(self, object, encoding, strict, decoded):
        self.encoded = object
        self.encoding = encoding
        self.strict = strict
        self.decoded = decoded

no_cached_decode = CachedDecode(None, None, None, None)


class Message(basetypes.Serializable):
    def __init__(self):
        self._content_cache = no_cached_decode  # type: CachedDecode
        self._text_cache = no_cached_decode  # type: CachedDecode

    def __eq__(self, other):
        if isinstance(other, Message):
            return self.data == other.data
        return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.data) ^ 1

    def get_state(self):
        return self.data.get_state()

    def set_state(self, state):
        self.data.set_state(state)

    @classmethod
    def from_state(cls, state):
        state["headers"] = headers.Headers.from_state(state["headers"])
        return cls(**state)

    @property
    def headers(self):
        """
        Message headers object

        Returns:
            netlib.http.Headers
        """
        return self.data.headers

    @headers.setter
    def headers(self, h):
        self.data.headers = h

    @property
    def raw_content(self):
        # type: () -> bytes
        """
        The raw (encoded) HTTP message body

        See also: :py:attr:`content`, :py:class:`text`
        """
        return self.data.content

    @raw_content.setter
    def raw_content(self, content):
        self.data.content = content

    def get_content(self, strict=True):
        # type: (bool) -> bytes
        """
        The HTTP message body decoded with the content-encoding header (e.g. gzip)

        Raises:
            ValueError, when the content-encoding is invalid and strict is True.

        See also: :py:class:`raw_content`, :py:attr:`text`
        """
        if self.raw_content is None:
            return None
        ce = self.headers.get("content-encoding")
        cached = (
            self._content_cache.encoded == self.raw_content and
            (self._content_cache.strict or not strict) and
            self._content_cache.encoding == ce
        )
        if not cached:
            is_strict = True
            if ce:
                try:
                    decoded = encoding.decode(self.raw_content, ce)
                except ValueError:
                    if strict:
                        raise
                    is_strict = False
                    decoded = self.raw_content
            else:
                decoded = self.raw_content
            self._content_cache = CachedDecode(self.raw_content, ce, is_strict, decoded)
        return self._content_cache.decoded

    def set_content(self, value):
        if value is None:
            self.raw_content = None
            return
        if not isinstance(value, bytes):
            raise TypeError(
                "Message content must be bytes, not {}. "
                "Please use .text if you want to assign a str."
                .format(type(value).__name__)
            )
        ce = self.headers.get("content-encoding")
        cached = (
            self._content_cache.decoded == value and
            self._content_cache.encoding == ce and
            self._content_cache.strict
        )
        if not cached:
            try:
                encoded = encoding.encode(value, ce or "identity")
            except ValueError:
                # So we have an invalid content-encoding?
                # Let's remove it!
                del self.headers["content-encoding"]
                ce = None
                encoded = value
            self._content_cache = CachedDecode(encoded, ce, True, value)
        self.raw_content = self._content_cache.encoded
        self.headers["content-length"] = str(len(self.raw_content))

    content = property(get_content, set_content)

    @property
    def http_version(self):
        """
        Version string, e.g. "HTTP/1.1"
        """
        return _native(self.data.http_version)

    @http_version.setter
    def http_version(self, http_version):
        self.data.http_version = _always_bytes(http_version)

    @property
    def timestamp_start(self):
        """
        First byte timestamp
        """
        return self.data.timestamp_start

    @timestamp_start.setter
    def timestamp_start(self, timestamp_start):
        self.data.timestamp_start = timestamp_start

    @property
    def timestamp_end(self):
        """
        Last byte timestamp
        """
        return self.data.timestamp_end

    @timestamp_end.setter
    def timestamp_end(self, timestamp_end):
        self.data.timestamp_end = timestamp_end

    def _get_content_type_charset(self):
        # type: () -> Optional[str]
        ct = headers.parse_content_type(self.headers.get("content-type", ""))
        if ct:
            return ct[2].get("charset")

    def _guess_encoding(self):
        # type: () -> str
        enc = self._get_content_type_charset()
        if enc:
            return enc

        if "json" in self.headers.get("content-type", ""):
            return "utf8"
        else:
            # We may also want to check for HTML meta tags here at some point.
            return "latin-1"

    def get_text(self, strict=True):
        # type: (bool) -> six.text_type
        """
        The HTTP message body decoded with both content-encoding header (e.g. gzip)
        and content-type header charset.

        Raises:
            ValueError, when either content-encoding or charset is invalid and strict is True.

        See also: :py:attr:`content`, :py:class:`raw_content`
        """
        if self.raw_content is None:
            return None
        enc = self._guess_encoding()

        content = self.get_content(strict)
        cached = (
            self._text_cache.encoded == content and
            (self._text_cache.strict or not strict) and
            self._text_cache.encoding == enc
        )
        if not cached:
            is_strict = self._content_cache.strict
            try:
                decoded = encoding.decode(content, enc)
            except ValueError:
                if strict:
                    raise
                is_strict = False
                decoded = self.content.decode("utf8", "replace" if six.PY2 else "surrogateescape")
            self._text_cache = CachedDecode(content, enc, is_strict, decoded)
        return self._text_cache.decoded

    def set_text(self, text):
        if text is None:
            self.content = None
            return
        enc = self._guess_encoding()

        cached = (
            self._text_cache.decoded == text and
            self._text_cache.encoding == enc and
            self._text_cache.strict
        )
        if not cached:
            try:
                encoded = encoding.encode(text, enc)
            except ValueError:
                # Fall back to UTF-8 and update the content-type header.
                ct = headers.parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {})
                ct[2]["charset"] = "utf-8"
                self.headers["content-type"] = headers.assemble_content_type(*ct)
                enc = "utf8"
                encoded = text.encode(enc, "replace" if six.PY2 else "surrogateescape")
            self._text_cache = CachedDecode(encoded, enc, True, text)
        self.content = self._text_cache.encoded

    text = property(get_text, set_text)

    def decode(self, strict=True):
        """
        Decodes body based on the current Content-Encoding header, then
        removes the header. If there is no Content-Encoding header, no
        action is taken.

        Raises:
            ValueError, when the content-encoding is invalid and strict is True.
        """
        self.raw_content = self.get_content(strict)
        self.headers.pop("content-encoding", None)

    def encode(self, e):
        """
        Encodes body with the encoding e, where e is "gzip", "deflate" or "identity".
        Any existing content-encodings are overwritten,
        the content is not decoded beforehand.

        Raises:
            ValueError, when the specified content-encoding is invalid.
        """
        self.headers["content-encoding"] = e
        self.content = self.raw_content
        if "content-encoding" not in self.headers:
            raise ValueError("Invalid content encoding {}".format(repr(e)))

    def replace(self, pattern, repl, flags=0):
        """
        Replaces a regular expression pattern with repl in both the headers
        and the body of the message. Encoded body will be decoded
        before replacement, and re-encoded afterwards.

        Returns:
            The number of replacements made.
        """
        if isinstance(pattern, six.text_type):
            pattern = strutils.escaped_str_to_bytes(pattern)
        if isinstance(repl, six.text_type):
            repl = strutils.escaped_str_to_bytes(repl)
        replacements = 0
        if self.content:
            self.content, replacements = re.subn(
                pattern, repl, self.content, flags=flags
            )
        replacements += self.headers.replace(pattern, repl, flags)
        return replacements

    # Legacy

    @property
    def body(self):  # pragma: no cover
        warnings.warn(".body is deprecated, use .content instead.", DeprecationWarning)
        return self.content

    @body.setter
    def body(self, body):  # pragma: no cover
        warnings.warn(".body is deprecated, use .content instead.", DeprecationWarning)
        self.content = body


class decoded(object):
    """
    Deprecated: You can now directly use :py:attr:`content`.
    :py:attr:`raw_content` has the encoded content.
    """

    def __init__(self, message):  # pragma no cover
        warnings.warn("decoded() is deprecated, you can now directly use .content instead. "
                      ".raw_content has the encoded content.", DeprecationWarning)

    def __enter__(self):  # pragma no cover
        pass

    def __exit__(self, type, value, tb):  # pragma no cover
        pass