Merge with master

docs/features/responsestreaming.rst

@@ -48,8 +48,7 @@ Implementation Details
 ----------------------
 
 When response streaming is enabled, portions of the code which would have otherwise performed
-changes on the response body will see an empty response body instead
-(:py:data:`netlib.http.CONTENT_MISSING`). Any modifications will be ignored.
+changes on the response body will see an empty response body. Any modifications will be ignored.
 
 Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a
 ``Transfer-Encoding: chunked`` header, the response will be streamed one chunk at a time.

mitmproxy/console/common.py

@@ -4,7 +4,6 @@ import urwid
 import urwid.util
 import os
 
-from netlib.http import CONTENT_MISSING
 import netlib.utils
 
 from .. import utils
@@ -256,7 +255,7 @@ def copy_flow_format_data(part, scope, flow):
     else:
         data = ""
         if scope in ("q", "a"):
-            if flow.request.content is None or flow.request.content == CONTENT_MISSING:
+            if flow.request.content is None:
                 return None, "Request content is missing"
             with decoded(flow.request):
                 if part == "h":
@@ -269,7 +268,7 @@ def copy_flow_format_data(part, scope, flow):
             # Add padding between request and response
             data += "\r\n" * 2
         if scope in ("s", "a") and flow.response:
-            if flow.response.content is None or flow.response.content == CONTENT_MISSING:
+            if flow.response.content is None:
                 return None, "Response content is missing"
             with decoded(flow.response):
                 if part == "h":
@@ -420,7 +419,7 @@ def format_flow(f, focus, extended=False, hostheader=False, marked=False):
     if f.response:
         if f.response.content:
             contentdesc = netlib.utils.pretty_size(len(f.response.content))
-        elif f.response.content == CONTENT_MISSING:
+        elif f.response.content is None:
             contentdesc = "[content missing]"
         else:
             contentdesc = "[no content]"

mitmproxy/console/flowview.py

@@ -7,7 +7,7 @@ import math
 import urwid
 
 from netlib import odict
-from netlib.http import CONTENT_MISSING, Headers
+from netlib.http import Headers
 from . import common, grideditor, signals, searchable, tabs
 from . import flowdetailview
 from .. import utils, controller, contentviews
@@ -169,7 +169,7 @@ class FlowView(tabs.Tabs):
             self.show()
 
     def content_view(self, viewmode, message):
-        if message.content == CONTENT_MISSING:
+        if message.content is None:
             msg, body = "", [urwid.Text([("error", "[content missing]")])]
             return msg, body
         else:
@@ -510,7 +510,7 @@ class FlowView(tabs.Tabs):
 
     def delete_body(self, t):
         if t == "m":
-            val = CONTENT_MISSING
+            val = None
         else:
             val = None
         if self.tab_offset == TAB_REQ:

mitmproxy/dump.py

@@ -5,7 +5,6 @@ import click
 import itertools
 
 from netlib import tcp
-from netlib.http import CONTENT_MISSING
 import netlib.utils
 from . import flow, filt, contentviews
 from .exceptions import ContentViewException
@@ -180,7 +179,7 @@ class DumpMaster(flow.FlowMaster):
             )
             self.echo(headers, indent=4)
         if self.o.flow_detail >= 3:
-            if message.content == CONTENT_MISSING:
+            if message.content is None:
                 self.echo("(content missing)", indent=4)
             elif message.content:
                 self.echo("")
@@ -283,7 +282,7 @@ class DumpMaster(flow.FlowMaster):
         code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418))
         reason = click.style(flow.response.reason, fg=code_color, bold=True)
 
-        if flow.response.content == CONTENT_MISSING:
+        if flow.response.content is None:
             size = "(content missing)"
         else:
             size = netlib.utils.pretty_size(len(flow.response.content))

mitmproxy/flow.py

@@ -16,7 +16,7 @@ from six.moves import urllib
 
 from netlib import wsgi
 from netlib.exceptions import HttpException
-from netlib.http import CONTENT_MISSING, Headers, http1
+from netlib.http import Headers, http1
 from . import controller, tnetstring, filt, script, version, flow_format_compat
 from .onboarding import app
 from .proxy.config import HostMatcher
@@ -942,7 +942,7 @@ class FlowMaster(controller.Master):
             return "Can't replay live request."
         if f.intercepted:
             return "Can't replay while intercepting..."
-        if f.request.content == CONTENT_MISSING:
+        if f.request.content is None:
             return "Can't replay request with missing content..."
         if f.request:
             f.backup()

mitmproxy/models/http.py

@@ -84,9 +84,8 @@ class HTTPRequest(MessageMixin, Request):
 
         headers: Headers object
 
-        content: Content of the request, None, or CONTENT_MISSING if there
-        is content associated, but not present. CONTENT_MISSING evaluates
-        to False to make checking for the presence of content natural.
+        content: Content of the request, the value is None if there is content
+        associated, but not present.
 
         form_in: The request form which mitmproxy has received. The following
         values are possible:
@@ -226,9 +225,8 @@ class HTTPResponse(MessageMixin, Response):
 
         headers: Headers object
 
-        content: Content of the request, None, or CONTENT_MISSING if there
-        is content associated, but not present. CONTENT_MISSING evaluates
-        to False to make checking for the presence of content natural.
+        content: Content of the response, the value is None if there is content
+        associated, but not present.
 
         timestamp_start: Timestamp indicating when request transmission started
 

mitmproxy/protocol/http.py

@@ -6,7 +6,7 @@ import six
 
 from netlib import tcp
 from netlib.exceptions import HttpException, HttpReadDisconnect, NetlibException
-from netlib.http import Headers, CONTENT_MISSING
+from netlib.http import Headers
 
 from h2.exceptions import H2Error
 
@@ -50,8 +50,8 @@ class _HttpTransmissionLayer(Layer):
         yield "this is a generator"  # pragma: no cover
 
     def send_response(self, response):
-        if response.content == CONTENT_MISSING:
-            raise HttpException("Cannot assemble flow with CONTENT_MISSING")
+        if response.content is None:
+            raise HttpException("Cannot assemble flow with missing content")
         self.send_response_headers(response)
         self.send_response_body(response, [response.content])
 
@@ -231,6 +231,7 @@ class HttpLayer(Layer):
                     six.reraise(ProtocolException, ProtocolException(
                         "Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
             finally:
+                if flow:
                     flow.live = False
 
     def get_request_from_client(self):
@@ -318,7 +319,7 @@ class HttpLayer(Layer):
             raise Kill()
 
         if flow.response.stream:
-            flow.response.data.content = CONTENT_MISSING
+            flow.response.data.content = None
         else:
             flow.response.data.content = b"".join(self.read_response_body(
                 flow.request,

mitmproxy/web/app.py

@@ -8,7 +8,6 @@ import logging
 import json
 import base64
 
-from netlib.http import CONTENT_MISSING
 from .. import version, filt
 
 
@@ -26,10 +25,8 @@ def _strip_content(flow_state):
                 continue
             if message["content"]:
                 message["contentLength"] = len(message["content"])
-            elif message["content"] == CONTENT_MISSING:
-                message["contentLength"] = None
             else:
-                message["contentLength"] = 0
+                message["contentLength"] = None
             del message["content"]
 
     if "backup" in flow_state:

netlib/http/__init__.py

@@ -2,13 +2,13 @@ from __future__ import absolute_import, print_function, division
 from .request import Request
 from .response import Response
 from .headers import Headers
-from .message import decoded, CONTENT_MISSING
+from .message import decoded
 from . import http1, http2
 
 __all__ = [
     "Request",
     "Response",
     "Headers",
-    "decoded", "CONTENT_MISSING",
+    "decoded",
     "http1", "http2",
 ]

netlib/http/http1/assemble.py

@@ -3,12 +3,10 @@ from __future__ import absolute_import, print_function, division
 from ... import utils
 import itertools
 from ...exceptions import HttpException
-from .. import CONTENT_MISSING
-
 
 def assemble_request(request):
-    if request.content == CONTENT_MISSING:
-        raise HttpException("Cannot assemble flow with CONTENT_MISSING")
+    if request.content is None:
+        raise HttpException("Cannot assemble flow with missing content")
     head = assemble_request_head(request)
     body = b"".join(assemble_body(request.data.headers, [request.data.content]))
     return head + body
@@ -21,8 +19,8 @@ def assemble_request_head(request):
 
 
 def assemble_response(response):
-    if response.content == CONTENT_MISSING:
-        raise HttpException("Cannot assemble flow with CONTENT_MISSING")
+    if response.content is None:
+        raise HttpException("Cannot assemble flow with missing content")
     head = assemble_response_head(response)
     body = b"".join(assemble_body(response.data.headers, [response.data.content]))
     return head + body

netlib/http/message.py

@@ -7,8 +7,6 @@ import six
 from .headers import Headers
 from .. import encoding, utils
 
-CONTENT_MISSING = 0
-
 if six.PY2:  # pragma: no cover
     _native = lambda x: x
     _always_bytes = lambda x: x

setup.cfg

@@ -9,7 +9,7 @@ ignore = E251
 
 [pytest]
 testpaths = test
-addopts = --timeout 30 -s
+addopts = --timeout 60 --capture=no
 
 [coverage:run]
 branch = True
@@ -19,6 +19,5 @@ omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py
 [coverage:report]
 show_missing = True
 exclude_lines =
-    pragma: nocover
     pragma: no cover
     raise NotImplementedError()

test/mitmproxy/test_dump.py

@@ -4,7 +4,6 @@ from mitmproxy.exceptions import ContentViewException
 from mitmproxy.models import HTTPResponse
 
 import netlib.tutils
-from netlib.http import CONTENT_MISSING
 
 from mitmproxy import dump, flow
 from mitmproxy.proxy import Log
@@ -38,7 +37,7 @@ def test_strfuncs():
     flow.request.stickycookie = True
     flow.client_conn = mock.MagicMock()
     flow.client_conn.address.host = "foo"
-    flow.response = netlib.tutils.tresp(content=CONTENT_MISSING)
+    flow.response = netlib.tutils.tresp(content=None)
     flow.response.is_replay = True
     flow.response.status_code = 300
     m.echo_flow(flow)
@@ -104,10 +103,10 @@ class TestDumpMaster:
         o = dump.Options(flow_detail=3)
         m = dump.DumpMaster(None, o, outfile=cs)
         f = tutils.tflow()
-        f.request.content = CONTENT_MISSING
+        f.request.content = None
         m.handle_request(f)
         f.response = HTTPResponse.wrap(netlib.tutils.tresp())
-        f.response.content = CONTENT_MISSING
+        f.response.content = None
         m.handle_response(f)
         assert "content missing" in cs.getvalue()
 

test/mitmproxy/test_flow.py

@@ -8,7 +8,7 @@ import mock
 
 import netlib.utils
 from netlib import odict
-from netlib.http import CONTENT_MISSING, Headers
+from netlib.http import Headers
 from mitmproxy import filt, controller, tnetstring, flow
 from mitmproxy.models import Error
 from mitmproxy.models import Flow
@@ -465,7 +465,7 @@ class TestFlow(object):
 
     def test_replace_no_content(self):
         f = tutils.tflow()
-        f.request.content = CONTENT_MISSING
+        f.request.content = None
         assert f.replace("foo", "bar") == 0
 
     def test_replace(self):
@@ -751,7 +751,7 @@ class TestFlowMaster:
         s = flow.State()
         fm = flow.FlowMaster(None, s)
         f = tutils.tflow(resp=True)
-        f.request.content = CONTENT_MISSING
+        f.request.content = None
         assert "missing" in fm.replay_request(f)
 
         f.intercepted = True

test/mitmproxy/test_flow_export.py

@@ -1,3 +1,4 @@
+import json
 from textwrap import dedent
 
 import netlib.tutils
@@ -178,6 +179,7 @@ class TestRawRequest():
         """).strip()
         assert flow_export.raw_request(flow) == result
 
+
 class TestExportLocustCode():
 
     def test_get(self):
@@ -369,3 +371,24 @@ class TestExportLocustTask():
         """.strip() + '\n'
 
         assert flow_export.locust_task(flow) == result
+
+
+class TestIsJson():
+
+    def test_empty(self):
+        assert flow_export.is_json(None, None) == False
+
+    def test_json_type(self):
+        headers = Headers(content_type="application/json")
+        assert flow_export.is_json(headers, "foobar") == False
+
+    def test_valid(self):
+        headers = Headers(content_type="application/foobar")
+        j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}')
+        assert j == False
+
+    def test_valid(self):
+        headers = Headers(content_type="application/json")
+        j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}')
+        assert isinstance(j, dict)
+

test/mitmproxy/test_server.py

@@ -8,7 +8,7 @@ from netlib.tcp import Address
 import netlib.tutils
 from netlib import tcp, http, socks
 from netlib.certutils import SSLCert
-from netlib.http import authentication, CONTENT_MISSING, http1
+from netlib.http import authentication, http1
 from netlib.tutils import raises
 from pathod import pathoc, pathod
 
@@ -281,7 +281,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin):
 
         self.pathod("200:b@3k")
         assert self.master.state.view[-1].response.stream
-        assert self.master.state.view[-1].response.content == CONTENT_MISSING
+        assert self.master.state.view[-1].response.content is None
         self.master.set_stream_large_bodies(None)
 
     def test_stream_modify(self):
@@ -816,7 +816,7 @@ class MasterIncomplete(tservers.TestMaster):
 
     def handle_request(self, f):
         resp = HTTPResponse.wrap(netlib.tutils.tresp())
-        resp.content = CONTENT_MISSING
+        resp.content = None
         f.reply(resp)
 
 

test/netlib/http/http1/test_assemble.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import, print_function, division
 from netlib.exceptions import HttpException
-from netlib.http import CONTENT_MISSING, Headers
+from netlib.http import Headers
 from netlib.http.http1.assemble import (
     assemble_request, assemble_request_head, assemble_response,
     assemble_response_head, _assemble_request_line, _assemble_request_headers,
@@ -20,7 +20,7 @@ def test_assemble_request():
     )
 
     with raises(HttpException):
-        assemble_request(treq(content=CONTENT_MISSING))
+        assemble_request(treq(content=None))
 
 
 def test_assemble_request_head():
@@ -41,7 +41,7 @@ def test_assemble_response():
     )
 
     with raises(HttpException):
-        assemble_response(tresp(content=CONTENT_MISSING))
+        assemble_response(tresp(content=None))
 
 
 def test_assemble_response_head():