Merge with master

docs/features/responsestreaming.rst

@@ -48,8 +48,7 @@ Implementation Details
 ----------------------
 
 When response streaming is enabled, portions of the code which would have otherwise performed
-changes on the response body will see an empty response body instead
+changes on the response body will see an empty response body. Any modifications will be ignored.
-(:py:data:`netlib.http.CONTENT_MISSING`). Any modifications will be ignored.
 
 Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a
 ``Transfer-Encoding: chunked`` header, the response will be streamed one chunk at a time.

mitmproxy/console/common.py

@@ -4,7 +4,6 @@ import urwid
 import urwid.util
 import os
 
-from netlib.http import CONTENT_MISSING
 import netlib.utils
 
 from .. import utils
@@ -256,7 +255,7 @@ def copy_flow_format_data(part, scope, flow):
     else:
         data = ""
         if scope in ("q", "a"):
-            if flow.request.content is None or flow.request.content == CONTENT_MISSING:
+            if flow.request.content is None:
                 return None, "Request content is missing"
             with decoded(flow.request):
                 if part == "h":
@@ -269,7 +268,7 @@ def copy_flow_format_data(part, scope, flow):
             # Add padding between request and response
             data += "\r\n" * 2
         if scope in ("s", "a") and flow.response:
-            if flow.response.content is None or flow.response.content == CONTENT_MISSING:
+            if flow.response.content is None:
                 return None, "Response content is missing"
             with decoded(flow.response):
                 if part == "h":
@@ -420,7 +419,7 @@ def format_flow(f, focus, extended=False, hostheader=False, marked=False):
     if f.response:
         if f.response.content:
             contentdesc = netlib.utils.pretty_size(len(f.response.content))
-        elif f.response.content == CONTENT_MISSING:
+        elif f.response.content is None:
             contentdesc = "[content missing]"
         else:
             contentdesc = "[no content]"

mitmproxy/console/flowview.py

@@ -7,7 +7,7 @@ import math
 import urwid
 
 from netlib import odict
-from netlib.http import CONTENT_MISSING, Headers
+from netlib.http import Headers
 from . import common, grideditor, signals, searchable, tabs
 from . import flowdetailview
 from .. import utils, controller, contentviews
@@ -169,7 +169,7 @@ class FlowView(tabs.Tabs):
             self.show()
 
     def content_view(self, viewmode, message):
-        if message.content == CONTENT_MISSING:
+        if message.content is None:
             msg, body = "", [urwid.Text([("error", "[content missing]")])]
             return msg, body
         else:
@@ -510,7 +510,7 @@ class FlowView(tabs.Tabs):
 
     def delete_body(self, t):
         if t == "m":
-            val = CONTENT_MISSING
+            val = None
         else:
             val = None
         if self.tab_offset == TAB_REQ:

mitmproxy/dump.py

@@ -5,7 +5,6 @@ import click
 import itertools
 
 from netlib import tcp
-from netlib.http import CONTENT_MISSING
 import netlib.utils
 from . import flow, filt, contentviews
 from .exceptions import ContentViewException
@@ -180,7 +179,7 @@ class DumpMaster(flow.FlowMaster):
             )
             self.echo(headers, indent=4)
         if self.o.flow_detail >= 3:
-            if message.content == CONTENT_MISSING:
+            if message.content is None:
                 self.echo("(content missing)", indent=4)
             elif message.content:
                 self.echo("")
@@ -283,7 +282,7 @@ class DumpMaster(flow.FlowMaster):
         code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418))
         reason = click.style(flow.response.reason, fg=code_color, bold=True)
 
-        if flow.response.content == CONTENT_MISSING:
+        if flow.response.content is None:
             size = "(content missing)"
         else:
             size = netlib.utils.pretty_size(len(flow.response.content))

mitmproxy/flow.py

@@ -16,7 +16,7 @@ from six.moves import urllib
 
 from netlib import wsgi
 from netlib.exceptions import HttpException
-from netlib.http import CONTENT_MISSING, Headers, http1
+from netlib.http import Headers, http1
 from . import controller, tnetstring, filt, script, version, flow_format_compat
 from .onboarding import app
 from .proxy.config import HostMatcher
@@ -942,7 +942,7 @@ class FlowMaster(controller.Master):
             return "Can't replay live request."
         if f.intercepted:
             return "Can't replay while intercepting..."
-        if f.request.content == CONTENT_MISSING:
+        if f.request.content is None:
             return "Can't replay request with missing content..."
         if f.request:
             f.backup()

mitmproxy/models/http.py

@@ -84,9 +84,8 @@ class HTTPRequest(MessageMixin, Request):
 
         headers: Headers object
 
-        content: Content of the request, None, or CONTENT_MISSING if there
+        content: Content of the request, the value is None if there is content
-        is content associated, but not present. CONTENT_MISSING evaluates
+        associated, but not present.
-        to False to make checking for the presence of content natural.
 
         form_in: The request form which mitmproxy has received. The following
         values are possible:
@@ -226,9 +225,8 @@ class HTTPResponse(MessageMixin, Response):
 
         headers: Headers object
 
-        content: Content of the request, None, or CONTENT_MISSING if there
+        content: Content of the response, the value is None if there is content
-        is content associated, but not present. CONTENT_MISSING evaluates
+        associated, but not present.
-        to False to make checking for the presence of content natural.
 
         timestamp_start: Timestamp indicating when request transmission started
 

mitmproxy/protocol/http.py

@@ -6,7 +6,7 @@ import six
 
 from netlib import tcp
 from netlib.exceptions import HttpException, HttpReadDisconnect, NetlibException
-from netlib.http import Headers, CONTENT_MISSING
+from netlib.http import Headers
 
 from h2.exceptions import H2Error
 
@@ -50,8 +50,8 @@ class _HttpTransmissionLayer(Layer):
         yield "this is a generator"  # pragma: no cover
 
     def send_response(self, response):
-        if response.content == CONTENT_MISSING:
+        if response.content is None:
-            raise HttpException("Cannot assemble flow with CONTENT_MISSING")
+            raise HttpException("Cannot assemble flow with missing content")
         self.send_response_headers(response)
         self.send_response_body(response, [response.content])
 
@@ -231,6 +231,7 @@ class HttpLayer(Layer):
                     six.reraise(ProtocolException, ProtocolException(
                         "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
             finally:
+                if flow:
                     flow.live = False
 
     def get_request_from_client(self):
@@ -318,7 +319,7 @@ class HttpLayer(Layer):
             raise Kill()
 
         if flow.response.stream:
-            flow.response.data.content = CONTENT_MISSING
+            flow.response.data.content = None
         else:
             flow.response.data.content = b"".join(self.read_response_body(
                 flow.request,

mitmproxy/script/script.py

@@ -47,7 +47,7 @@ class Script(object):
         if os.name == "nt":  # pragma: no cover
             backslashes = shlex.split(command, posix=False)[0].count("\\")
             command = command.replace("\\", "\\\\", backslashes)
-        args = shlex.split(command)  # pragma: nocover
+        args = shlex.split(command)  # pragma: no cover
         args[0] = os.path.expanduser(args[0])
         if not os.path.exists(args[0]):
             raise ScriptException(

mitmproxy/web/app.py

@@ -8,7 +8,6 @@ import logging
 import json
 import base64
 
-from netlib.http import CONTENT_MISSING
 from .. import version, filt
 
 
@@ -26,10 +25,8 @@ def _strip_content(flow_state):
                 continue
             if message["content"]:
                 message["contentLength"] = len(message["content"])
-            elif message["content"] == CONTENT_MISSING:
-                message["contentLength"] = None
             else:
-                message["contentLength"] = 0
+                message["contentLength"] = None
             del message["content"]
 
     if "backup" in flow_state:

netlib/http/__init__.py

@@ -2,13 +2,13 @@ from __future__ import absolute_import, print_function, division
 from .request import Request
 from .response import Response
 from .headers import Headers
-from .message import decoded, CONTENT_MISSING
+from .message import decoded
 from . import http1, http2
 
 __all__ = [
     "Request",
     "Response",
     "Headers",
-    "decoded", "CONTENT_MISSING",
+    "decoded",
     "http1", "http2",
 ]

netlib/http/authentication.py

@@ -140,7 +140,7 @@ class AuthAction(Action):
         authenticator = BasicProxyAuth(passman, "mitmproxy")
         setattr(namespace, self.dest, authenticator)
 
-    def getPasswordManager(self, s):  # pragma: nocover
+    def getPasswordManager(self, s):  # pragma: no cover
         raise NotImplementedError()
 
 

netlib/http/headers.py

@@ -8,7 +8,7 @@ from __future__ import absolute_import, print_function, division
 import copy
 try:
     from collections.abc import MutableMapping
-except ImportError:  # pragma: nocover
+except ImportError:  # pragma: no cover
     from collections import MutableMapping  # Workaround for Python < 3.3
 
 
@@ -16,7 +16,7 @@ import six
 
 from netlib.utils import always_byte_args, always_bytes, Serializable
 
-if six.PY2:  # pragma: nocover
+if six.PY2:  # pragma: no cover
     _native = lambda x: x
     _always_bytes = lambda x: x
     _always_byte_args = lambda x: x
@@ -106,7 +106,7 @@ class Headers(MutableMapping, Serializable):
         else:
             return b""
 
-    if six.PY2:  # pragma: nocover
+    if six.PY2:  # pragma: no cover
         __str__ = __bytes__
 
     @_always_byte_args

netlib/http/http1/assemble.py

@@ -3,12 +3,10 @@ from __future__ import absolute_import, print_function, division
 from ... import utils
 import itertools
 from ...exceptions import HttpException
-from .. import CONTENT_MISSING
-
 
 def assemble_request(request):
-    if request.content == CONTENT_MISSING:
+    if request.content is None:
-        raise HttpException("Cannot assemble flow with CONTENT_MISSING")
+        raise HttpException("Cannot assemble flow with missing content")
     head = assemble_request_head(request)
     body = b"".join(assemble_body(request.data.headers, [request.data.content]))
     return head + body
@@ -21,8 +19,8 @@ def assemble_request_head(request):
 
 
 def assemble_response(response):
-    if response.content == CONTENT_MISSING:
+    if response.content is None:
-        raise HttpException("Cannot assemble flow with CONTENT_MISSING")
+        raise HttpException("Cannot assemble flow with missing content")
     head = assemble_response_head(response)
     body = b"".join(assemble_body(response.data.headers, [response.data.content]))
     return head + body

netlib/http/message.py

@@ -7,9 +7,7 @@ import six
 from .headers import Headers
 from .. import encoding, utils
 
-CONTENT_MISSING = 0
+if six.PY2:  # pragma: no cover
-
-if six.PY2:  # pragma: nocover
     _native = lambda x: x
     _always_bytes = lambda x: x
 else:
@@ -182,12 +180,12 @@ class Message(utils.Serializable):
     # Legacy
 
     @property
-    def body(self):  # pragma: nocover
+    def body(self):  # pragma: no cover
         warnings.warn(".body is deprecated, use .content instead.", DeprecationWarning)
         return self.content
 
     @body.setter
-    def body(self, body):  # pragma: nocover
+    def body(self, body):  # pragma: no cover
         warnings.warn(".body is deprecated, use .content instead.", DeprecationWarning)
         self.content = body
 

netlib/http/request.py

@@ -100,7 +100,7 @@ class Request(Message):
         Setting the host attribute also updates the host header, if present.
         """
 
-        if six.PY2:  # pragma: nocover
+        if six.PY2:  # pragma: no cover
             return self.data.host
 
         if not self.data.host:
@@ -324,59 +324,59 @@ class Request(Message):
 
     # Legacy
 
-    def get_cookies(self):  # pragma: nocover
+    def get_cookies(self):  # pragma: no cover
         warnings.warn(".get_cookies is deprecated, use .cookies instead.", DeprecationWarning)
         return self.cookies
 
-    def set_cookies(self, odict):  # pragma: nocover
+    def set_cookies(self, odict):  # pragma: no cover
         warnings.warn(".set_cookies is deprecated, use .cookies instead.", DeprecationWarning)
         self.cookies = odict
 
-    def get_query(self):  # pragma: nocover
+    def get_query(self):  # pragma: no cover
         warnings.warn(".get_query is deprecated, use .query instead.", DeprecationWarning)
         return self.query or ODict([])
 
-    def set_query(self, odict):  # pragma: nocover
+    def set_query(self, odict):  # pragma: no cover
         warnings.warn(".set_query is deprecated, use .query instead.", DeprecationWarning)
         self.query = odict
 
-    def get_path_components(self):  # pragma: nocover
+    def get_path_components(self):  # pragma: no cover
         warnings.warn(".get_path_components is deprecated, use .path_components instead.", DeprecationWarning)
         return self.path_components
 
-    def set_path_components(self, lst):  # pragma: nocover
+    def set_path_components(self, lst):  # pragma: no cover
         warnings.warn(".set_path_components is deprecated, use .path_components instead.", DeprecationWarning)
         self.path_components = lst
 
-    def get_form_urlencoded(self):  # pragma: nocover
+    def get_form_urlencoded(self):  # pragma: no cover
         warnings.warn(".get_form_urlencoded is deprecated, use .urlencoded_form instead.", DeprecationWarning)
         return self.urlencoded_form or ODict([])
 
-    def set_form_urlencoded(self, odict):  # pragma: nocover
+    def set_form_urlencoded(self, odict):  # pragma: no cover
         warnings.warn(".set_form_urlencoded is deprecated, use .urlencoded_form instead.", DeprecationWarning)
         self.urlencoded_form = odict
 
-    def get_form_multipart(self):  # pragma: nocover
+    def get_form_multipart(self):  # pragma: no cover
         warnings.warn(".get_form_multipart is deprecated, use .multipart_form instead.", DeprecationWarning)
         return self.multipart_form or ODict([])
 
     @property
-    def form_in(self):  # pragma: nocover
+    def form_in(self):  # pragma: no cover
         warnings.warn(".form_in is deprecated, use .first_line_format instead.", DeprecationWarning)
         return self.first_line_format
 
     @form_in.setter
-    def form_in(self, form_in):  # pragma: nocover
+    def form_in(self, form_in):  # pragma: no cover
         warnings.warn(".form_in is deprecated, use .first_line_format instead.", DeprecationWarning)
         self.first_line_format = form_in
 
     @property
-    def form_out(self):  # pragma: nocover
+    def form_out(self):  # pragma: no cover
         warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning)
         return self.first_line_format
 
     @form_out.setter
-    def form_out(self, form_out):  # pragma: nocover
+    def form_out(self, form_out):  # pragma: no cover
         warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning)
         self.first_line_format = form_out
 

netlib/http/response.py

@@ -97,20 +97,20 @@ class Response(Message):
 
     # Legacy
 
-    def get_cookies(self):  # pragma: nocover
+    def get_cookies(self):  # pragma: no cover
         warnings.warn(".get_cookies is deprecated, use .cookies instead.", DeprecationWarning)
         return self.cookies
 
-    def set_cookies(self, odict):  # pragma: nocover
+    def set_cookies(self, odict):  # pragma: no cover
         warnings.warn(".set_cookies is deprecated, use .cookies instead.", DeprecationWarning)
         self.cookies = odict
 
     @property
-    def msg(self):  # pragma: nocover
+    def msg(self):  # pragma: no cover
         warnings.warn(".msg is deprecated, use .reason instead.", DeprecationWarning)
         return self.reason
 
     @msg.setter
-    def msg(self, reason):  # pragma: nocover
+    def msg(self, reason):  # pragma: no cover
         warnings.warn(".msg is deprecated, use .reason instead.", DeprecationWarning)
         self.reason = reason

pathod/pathoc.py

@@ -208,7 +208,7 @@ class Pathoc(tcp.TCPClient):
         self.ws_framereader = None
 
         if self.use_http2:
-            if not tcp.HAS_ALPN:  # pragma: nocover
+            if not tcp.HAS_ALPN:  # pragma: no cover
                 log.write_raw(
                     self.fp,
                     "HTTP/2 requires ALPN support. "
@@ -458,7 +458,7 @@ class Pathoc(tcp.TCPClient):
             # TODO: do something
 
 
-def main(args):  # pragma: nocover
+def main(args):  # pragma: no cover
     memo = set([])
     trycount = 0
     p = None

pathod/pathoc_cmdline.py

@@ -221,6 +221,6 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
     return args
 
 
-def go_pathoc():  # pragma: nocover
+def go_pathoc():  # pragma: no cover
     args = args_pathoc(sys.argv)
     pathoc.main(args)

pathod/pathod.py

@@ -430,7 +430,7 @@ class Pathod(tcp.TCPServer):
         return self.log
 
 
-def main(args):  # pragma: nocover
+def main(args):  # pragma: no cover
     ssloptions = SSLOptions(
         cn=args.cn,
         confdir=args.confdir,

pathod/pathod_cmdline.py

@@ -226,6 +226,6 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
     return args
 
 
-def go_pathod():  # pragma: nocover
+def go_pathod():  # pragma: no cover
     args = args_pathod(sys.argv)
     pathod.main(args)

pathod/utils.py

@@ -98,7 +98,7 @@ class Data(object):
 data = Data(__name__)
 
 
-def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):  # pragma: nocover
+def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):  # pragma: no cover
     try:
         pid = os.fork()
         if pid > 0:

setup.cfg

@@ -9,7 +9,7 @@ ignore = E251
 
 [pytest]
 testpaths = test
-addopts = --timeout 30 -s
+addopts = --timeout 60 --capture=no
 
 [coverage:run]
 branch = True
@@ -19,6 +19,5 @@ omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py
 [coverage:report]
 show_missing = True
 exclude_lines =
-    pragma: nocover
     pragma: no cover
     raise NotImplementedError()

test/mitmproxy/test_dump.py

@@ -4,7 +4,6 @@ from mitmproxy.exceptions import ContentViewException
 from mitmproxy.models import HTTPResponse
 
 import netlib.tutils
-from netlib.http import CONTENT_MISSING
 
 from mitmproxy import dump, flow
 from mitmproxy.proxy import Log
@@ -38,7 +37,7 @@ def test_strfuncs():
     flow.request.stickycookie = True
     flow.client_conn = mock.MagicMock()
     flow.client_conn.address.host = "foo"
-    flow.response = netlib.tutils.tresp(content=CONTENT_MISSING)
+    flow.response = netlib.tutils.tresp(content=None)
     flow.response.is_replay = True
     flow.response.status_code = 300
     m.echo_flow(flow)
@@ -104,10 +103,10 @@ class TestDumpMaster:
         o = dump.Options(flow_detail=3)
         m = dump.DumpMaster(None, o, outfile=cs)
         f = tutils.tflow()
-        f.request.content = CONTENT_MISSING
+        f.request.content = None
         m.handle_request(f)
         f.response = HTTPResponse.wrap(netlib.tutils.tresp())
-        f.response.content = CONTENT_MISSING
+        f.response.content = None
         m.handle_response(f)
         assert "content missing" in cs.getvalue()
 

test/mitmproxy/test_flow.py

@@ -8,7 +8,7 @@ import mock
 
 import netlib.utils
 from netlib import odict
-from netlib.http import CONTENT_MISSING, Headers
+from netlib.http import Headers
 from mitmproxy import filt, controller, tnetstring, flow
 from mitmproxy.models import Error
 from mitmproxy.models import Flow
@@ -465,7 +465,7 @@ class TestFlow(object):
 
     def test_replace_no_content(self):
         f = tutils.tflow()
-        f.request.content = CONTENT_MISSING
+        f.request.content = None
         assert f.replace("foo", "bar") == 0
 
     def test_replace(self):
@@ -751,7 +751,7 @@ class TestFlowMaster:
         s = flow.State()
         fm = flow.FlowMaster(None, s)
         f = tutils.tflow(resp=True)
-        f.request.content = CONTENT_MISSING
+        f.request.content = None
         assert "missing" in fm.replay_request(f)
 
         f.intercepted = True

test/mitmproxy/test_flow_export.py

@@ -1,3 +1,4 @@
+import json
 from textwrap import dedent
 
 import netlib.tutils
@@ -178,6 +179,7 @@ class TestRawRequest():
         """).strip()
         assert flow_export.raw_request(flow) == result
 
+
 class TestExportLocustCode():
 
     def test_get(self):
@@ -369,3 +371,24 @@ class TestExportLocustTask():
         """.strip() + '\n'
 
         assert flow_export.locust_task(flow) == result
+
+
+class TestIsJson():
+
+    def test_empty(self):
+        assert flow_export.is_json(None, None) == False
+
+    def test_json_type(self):
+        headers = Headers(content_type="application/json")
+        assert flow_export.is_json(headers, "foobar") == False
+
+    def test_valid(self):
+        headers = Headers(content_type="application/foobar")
+        j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}')
+        assert j == False
+
+    def test_valid(self):
+        headers = Headers(content_type="application/json")
+        j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}')
+        assert isinstance(j, dict)
+

test/mitmproxy/test_server.py

@@ -8,7 +8,7 @@ from netlib.tcp import Address
 import netlib.tutils
 from netlib import tcp, http, socks
 from netlib.certutils import SSLCert
-from netlib.http import authentication, CONTENT_MISSING, http1
+from netlib.http import authentication, http1
 from netlib.tutils import raises
 from pathod import pathoc, pathod
 
@@ -281,7 +281,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin):
 
         self.pathod("200:b@3k")
         assert self.master.state.view[-1].response.stream
-        assert self.master.state.view[-1].response.content == CONTENT_MISSING
+        assert self.master.state.view[-1].response.content is None
         self.master.set_stream_large_bodies(None)
 
     def test_stream_modify(self):
@@ -816,7 +816,7 @@ class MasterIncomplete(tservers.TestMaster):
 
     def handle_request(self, f):
         resp = HTTPResponse.wrap(netlib.tutils.tresp())
-        resp.content = CONTENT_MISSING
+        resp.content = None
         f.reply(resp)
 
 

test/netlib/http/http1/test_assemble.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import, print_function, division
 from netlib.exceptions import HttpException
-from netlib.http import CONTENT_MISSING, Headers
+from netlib.http import Headers
 from netlib.http.http1.assemble import (
     assemble_request, assemble_request_head, assemble_response,
     assemble_response_head, _assemble_request_line, _assemble_request_headers,
@@ -20,7 +20,7 @@ def test_assemble_request():
     )
 
     with raises(HttpException):
-        assemble_request(treq(content=CONTENT_MISSING))
+        assemble_request(treq(content=None))
 
 
 def test_assemble_request_head():
@@ -41,7 +41,7 @@ def test_assemble_response():
     )
 
     with raises(HttpException):
-        assemble_response(tresp(content=CONTENT_MISSING))
+        assemble_response(tresp(content=None))
 
 
 def test_assemble_response_head():