Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-27 02:24:18 +00:00
Code Issues Projects Releases Wiki Activity

Merge with master

Browse Source
This commit is contained in:
Zohar Lorberbaum 2016-03-28 10:03:26 -07:00
parent e56198ae7c
commit 6d16f44ab7
27 changed files with 96 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/features/responsestreaming.rst
View File

@ -48,8 +48,7 @@ Implementation Details
----------------------
When response streaming is enabled, portions of the code which would have otherwise performed
changes on the response body will see an empty response body instead
(:py:data:`netlib.http.CONTENT_MISSING`). Any modifications will be ignored.
changes on the response body will see an empty response body. Any modifications will be ignored.
Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a
``Transfer-Encoding: chunked`` header, the response will be streamed one chunk at a time.

7
mitmproxy/console/common.py
View File

@ -4,7 +4,6 @@ import urwid
import urwid.util
import os
from netlib.http import CONTENT_MISSING
import netlib.utils
from .. import utils
@ -256,7 +255,7 @@ def copy_flow_format_data(part, scope, flow):
else:
data = ""
if scope in ("q", "a"):
if flow.request.content is None or flow.request.content == CONTENT_MISSING:
if flow.request.content is None:
return None, "Request content is missing"
with decoded(flow.request):
if part == "h":
@ -269,7 +268,7 @@ def copy_flow_format_data(part, scope, flow):
# Add padding between request and response
data += "\r\n" * 2
if scope in ("s", "a") and flow.response:
if flow.response.content is None or flow.response.content == CONTENT_MISSING:
if flow.response.content is None:
return None, "Response content is missing"
with decoded(flow.response):
if part == "h":
@ -420,7 +419,7 @@ def format_flow(f, focus, extended=False, hostheader=False, marked=False):
if f.response:
if f.response.content:
contentdesc = netlib.utils.pretty_size(len(f.response.content))
elif f.response.content == CONTENT_MISSING:
elif f.response.content is None:
contentdesc = "[content missing]"
else:
contentdesc = "[no content]"

6
mitmproxy/console/flowview.py
View File

@ -7,7 +7,7 @@ import math
import urwid
from netlib import odict
from netlib.http import CONTENT_MISSING, Headers
from netlib.http import Headers
from . import common, grideditor, signals, searchable, tabs
from . import flowdetailview
from .. import utils, controller, contentviews
@ -169,7 +169,7 @@ class FlowView(tabs.Tabs):
self.show()
def content_view(self, viewmode, message):
if message.content == CONTENT_MISSING:
if message.content is None:
msg, body = "", [urwid.Text([("error", "[content missing]")])]
return msg, body
else:
@ -510,7 +510,7 @@ class FlowView(tabs.Tabs):
def delete_body(self, t):
if t == "m":
val = CONTENT_MISSING
val = None
else:
val = None
if self.tab_offset == TAB_REQ:

5
mitmproxy/dump.py
View File

@ -5,7 +5,6 @@ import click
import itertools
from netlib import tcp
from netlib.http import CONTENT_MISSING
import netlib.utils
from . import flow, filt, contentviews
from .exceptions import ContentViewException
@ -180,7 +179,7 @@ class DumpMaster(flow.FlowMaster):
)
self.echo(headers, indent=4)
if self.o.flow_detail >= 3:
if message.content == CONTENT_MISSING:
if message.content is None:
self.echo("(content missing)", indent=4)
elif message.content:
self.echo("")
@ -283,7 +282,7 @@ class DumpMaster(flow.FlowMaster):
code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418))
reason = click.style(flow.response.reason, fg=code_color, bold=True)
if flow.response.content == CONTENT_MISSING:
if flow.response.content is None:
size = "(content missing)"
else:
size = netlib.utils.pretty_size(len(flow.response.content))

4
mitmproxy/flow.py
View File

@ -16,7 +16,7 @@ from six.moves import urllib
from netlib import wsgi
from netlib.exceptions import HttpException
from netlib.http import CONTENT_MISSING, Headers, http1
from netlib.http import Headers, http1
from . import controller, tnetstring, filt, script, version, flow_format_compat
from .onboarding import app
from .proxy.config import HostMatcher
@ -942,7 +942,7 @@ class FlowMaster(controller.Master):
return "Can't replay live request."
if f.intercepted:
return "Can't replay while intercepting..."
if f.request.content == CONTENT_MISSING:
if f.request.content is None:
return "Can't replay request with missing content..."
if f.request:
f.backup()

10
mitmproxy/models/http.py
View File

@ -84,9 +84,8 @@ class HTTPRequest(MessageMixin, Request):
headers: Headers object
content: Content of the request, None, or CONTENT_MISSING if there
is content associated, but not present. CONTENT_MISSING evaluates
to False to make checking for the presence of content natural.
content: Content of the request, the value is None if there is content
associated, but not present.
form_in: The request form which mitmproxy has received. The following
values are possible:
@ -226,9 +225,8 @@ class HTTPResponse(MessageMixin, Response):
headers: Headers object
content: Content of the request, None, or CONTENT_MISSING if there
is content associated, but not present. CONTENT_MISSING evaluates
to False to make checking for the presence of content natural.
content: Content of the response, the value is None if there is content
associated, but not present.
timestamp_start: Timestamp indicating when request transmission started

9
mitmproxy/protocol/http.py
View File

@ -6,7 +6,7 @@ import six
from netlib import tcp
from netlib.exceptions import HttpException, HttpReadDisconnect, NetlibException
from netlib.http import Headers, CONTENT_MISSING
from netlib.http import Headers
from h2.exceptions import H2Error
@ -50,8 +50,8 @@ class _HttpTransmissionLayer(Layer):
yield "this is a generator" # pragma: no cover
def send_response(self, response):
if response.content == CONTENT_MISSING:
raise HttpException("Cannot assemble flow with CONTENT_MISSING")
if response.content is None:
raise HttpException("Cannot assemble flow with missing content")
self.send_response_headers(response)
self.send_response_body(response, [response.content])
@ -231,6 +231,7 @@ class HttpLayer(Layer):
six.reraise(ProtocolException, ProtocolException(
"Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
finally:
if flow:
flow.live = False
def get_request_from_client(self):
@ -318,7 +319,7 @@ class HttpLayer(Layer):
raise Kill()
if flow.response.stream:
flow.response.data.content = CONTENT_MISSING
flow.response.data.content = None
else:
flow.response.data.content = b"".join(self.read_response_body(
flow.request,

2
mitmproxy/script/script.py
View File

@ -47,7 +47,7 @@ class Script(object):
if os.name == "nt": # pragma: no cover
backslashes = shlex.split(command, posix=False)[0].count("\\")
command = command.replace("\\", "\\\\", backslashes)
args = shlex.split(command) # pragma: nocover
args = shlex.split(command) # pragma: no cover
args[0] = os.path.expanduser(args[0])
if not os.path.exists(args[0]):
raise ScriptException(

5
mitmproxy/web/app.py
View File

@ -8,7 +8,6 @@ import logging
import json
import base64
from netlib.http import CONTENT_MISSING
from .. import version, filt
@ -26,10 +25,8 @@ def _strip_content(flow_state):
continue
if message["content"]:
message["contentLength"] = len(message["content"])
elif message["content"] == CONTENT_MISSING:
message["contentLength"] = None
else:
message["contentLength"] = 0
message["contentLength"] = None
del message["content"]
if "backup" in flow_state:

4
netlib/http/__init__.py
View File

@ -2,13 +2,13 @@ from __future__ import absolute_import, print_function, division
from .request import Request
from .response import Response
from .headers import Headers
from .message import decoded, CONTENT_MISSING
from .message import decoded
from . import http1, http2
__all__ = [
"Request",
"Response",
"Headers",
"decoded", "CONTENT_MISSING",
"decoded",
"http1", "http2",
]

2
netlib/http/authentication.py
View File

@ -140,7 +140,7 @@ class AuthAction(Action):
authenticator = BasicProxyAuth(passman, "mitmproxy")
setattr(namespace, self.dest, authenticator)
def getPasswordManager(self, s): # pragma: nocover
def getPasswordManager(self, s): # pragma: no cover
raise NotImplementedError()

6
netlib/http/headers.py
View File

@ -8,7 +8,7 @@ from __future__ import absolute_import, print_function, division
import copy
try:
from collections.abc import MutableMapping
except ImportError: # pragma: nocover
except ImportError: # pragma: no cover
from collections import MutableMapping # Workaround for Python < 3.3
@ -16,7 +16,7 @@ import six
from netlib.utils import always_byte_args, always_bytes, Serializable
if six.PY2: # pragma: nocover
if six.PY2: # pragma: no cover
_native = lambda x: x
_always_bytes = lambda x: x
_always_byte_args = lambda x: x
@ -106,7 +106,7 @@ class Headers(MutableMapping, Serializable):
else:
return b""
if six.PY2: # pragma: nocover
if six.PY2: # pragma: no cover
__str__ = __bytes__
@_always_byte_args

10
netlib/http/http1/assemble.py
View File

@ -3,12 +3,10 @@ from __future__ import absolute_import, print_function, division
from ... import utils
import itertools
from ...exceptions import HttpException
from .. import CONTENT_MISSING
def assemble_request(request):
if request.content == CONTENT_MISSING:
raise HttpException("Cannot assemble flow with CONTENT_MISSING")
if request.content is None:
raise HttpException("Cannot assemble flow with missing content")
head = assemble_request_head(request)
body = b"".join(assemble_body(request.data.headers, [request.data.content]))
return head + body
@ -21,8 +19,8 @@ def assemble_request_head(request):
def assemble_response(response):
if response.content == CONTENT_MISSING:
raise HttpException("Cannot assemble flow with CONTENT_MISSING")
if response.content is None:
raise HttpException("Cannot assemble flow with missing content")
head = assemble_response_head(response)
body = b"".join(assemble_body(response.data.headers, [response.data.content]))
return head + body

8
netlib/http/message.py
View File

@ -7,9 +7,7 @@ import six
from .headers import Headers
from .. import encoding, utils
CONTENT_MISSING = 0
if six.PY2: # pragma: nocover
if six.PY2: # pragma: no cover
_native = lambda x: x
_always_bytes = lambda x: x
else:
@ -182,12 +180,12 @@ class Message(utils.Serializable):
# Legacy
@property
def body(self): # pragma: nocover
def body(self): # pragma: no cover
warnings.warn(".body is deprecated, use .content instead.", DeprecationWarning)
return self.content
@body.setter
def body(self, body): # pragma: nocover
def body(self, body): # pragma: no cover
warnings.warn(".body is deprecated, use .content instead.", DeprecationWarning)
self.content = body

28
netlib/http/request.py
View File

@ -100,7 +100,7 @@ class Request(Message):
Setting the host attribute also updates the host header, if present.
"""
if six.PY2: # pragma: nocover
if six.PY2: # pragma: no cover
return self.data.host
if not self.data.host:
@ -324,59 +324,59 @@ class Request(Message):
# Legacy
def get_cookies(self): # pragma: nocover
def get_cookies(self): # pragma: no cover
warnings.warn(".get_cookies is deprecated, use .cookies instead.", DeprecationWarning)
return self.cookies
def set_cookies(self, odict): # pragma: nocover
def set_cookies(self, odict): # pragma: no cover
warnings.warn(".set_cookies is deprecated, use .cookies instead.", DeprecationWarning)
self.cookies = odict
def get_query(self): # pragma: nocover
def get_query(self): # pragma: no cover
warnings.warn(".get_query is deprecated, use .query instead.", DeprecationWarning)
return self.query or ODict([])
def set_query(self, odict): # pragma: nocover
def set_query(self, odict): # pragma: no cover
warnings.warn(".set_query is deprecated, use .query instead.", DeprecationWarning)
self.query = odict
def get_path_components(self): # pragma: nocover
def get_path_components(self): # pragma: no cover
warnings.warn(".get_path_components is deprecated, use .path_components instead.", DeprecationWarning)
return self.path_components
def set_path_components(self, lst): # pragma: nocover
def set_path_components(self, lst): # pragma: no cover
warnings.warn(".set_path_components is deprecated, use .path_components instead.", DeprecationWarning)
self.path_components = lst
def get_form_urlencoded(self): # pragma: nocover
def get_form_urlencoded(self): # pragma: no cover
warnings.warn(".get_form_urlencoded is deprecated, use .urlencoded_form instead.", DeprecationWarning)
return self.urlencoded_form or ODict([])
def set_form_urlencoded(self, odict): # pragma: nocover
def set_form_urlencoded(self, odict): # pragma: no cover
warnings.warn(".set_form_urlencoded is deprecated, use .urlencoded_form instead.", DeprecationWarning)
self.urlencoded_form = odict
def get_form_multipart(self): # pragma: nocover
def get_form_multipart(self): # pragma: no cover
warnings.warn(".get_form_multipart is deprecated, use .multipart_form instead.", DeprecationWarning)
return self.multipart_form or ODict([])
@property
def form_in(self): # pragma: nocover
def form_in(self): # pragma: no cover
warnings.warn(".form_in is deprecated, use .first_line_format instead.", DeprecationWarning)
return self.first_line_format
@form_in.setter
def form_in(self, form_in): # pragma: nocover
def form_in(self, form_in): # pragma: no cover
warnings.warn(".form_in is deprecated, use .first_line_format instead.", DeprecationWarning)
self.first_line_format = form_in
@property
def form_out(self): # pragma: nocover
def form_out(self): # pragma: no cover
warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning)
return self.first_line_format
@form_out.setter
def form_out(self, form_out): # pragma: nocover
def form_out(self, form_out): # pragma: no cover
warnings.warn(".form_out is deprecated, use .first_line_format instead.", DeprecationWarning)
self.first_line_format = form_out

8
netlib/http/response.py
View File

@ -97,20 +97,20 @@ class Response(Message):
# Legacy
def get_cookies(self): # pragma: nocover
def get_cookies(self): # pragma: no cover
warnings.warn(".get_cookies is deprecated, use .cookies instead.", DeprecationWarning)
return self.cookies
def set_cookies(self, odict): # pragma: nocover
def set_cookies(self, odict): # pragma: no cover
warnings.warn(".set_cookies is deprecated, use .cookies instead.", DeprecationWarning)
self.cookies = odict
@property
def msg(self): # pragma: nocover
def msg(self): # pragma: no cover
warnings.warn(".msg is deprecated, use .reason instead.", DeprecationWarning)
return self.reason
@msg.setter
def msg(self, reason): # pragma: nocover
def msg(self, reason): # pragma: no cover
warnings.warn(".msg is deprecated, use .reason instead.", DeprecationWarning)
self.reason = reason

4
pathod/pathoc.py
View File

@ -208,7 +208,7 @@ class Pathoc(tcp.TCPClient):
self.ws_framereader = None
if self.use_http2:
if not tcp.HAS_ALPN: # pragma: nocover
if not tcp.HAS_ALPN: # pragma: no cover
log.write_raw(
self.fp,
"HTTP/2 requires ALPN support. "
@ -458,7 +458,7 @@ class Pathoc(tcp.TCPClient):
# TODO: do something
def main(args): # pragma: nocover
def main(args): # pragma: no cover
memo = set([])
trycount = 0
p = None

2
pathod/pathoc_cmdline.py
View File

@ -221,6 +221,6 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
return args
def go_pathoc(): # pragma: nocover
def go_pathoc(): # pragma: no cover
args = args_pathoc(sys.argv)
pathoc.main(args)

2
pathod/pathod.py
View File

@ -430,7 +430,7 @@ class Pathod(tcp.TCPServer):
return self.log
def main(args): # pragma: nocover
def main(args): # pragma: no cover
ssloptions = SSLOptions(
cn=args.cn,
confdir=args.confdir,

2
pathod/pathod_cmdline.py
View File

@ -226,6 +226,6 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
return args
def go_pathod(): # pragma: nocover
def go_pathod(): # pragma: no cover
args = args_pathod(sys.argv)
pathod.main(args)

2
pathod/utils.py
View File

@ -98,7 +98,7 @@ class Data(object):
data = Data(__name__)
def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): # pragma: nocover
def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): # pragma: no cover
try:
pid = os.fork()
if pid > 0:

3
setup.cfg
View File

@ -9,7 +9,7 @@ ignore = E251
[pytest]
testpaths = test
addopts = --timeout 30 -s
addopts = --timeout 60 --capture=no
[coverage:run]
branch = True
@ -19,6 +19,5 @@ omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py
[coverage:report]
show_missing = True
exclude_lines =
pragma: nocover
pragma: no cover
raise NotImplementedError()

7
test/mitmproxy/test_dump.py
View File

@ -4,7 +4,6 @@ from mitmproxy.exceptions import ContentViewException
from mitmproxy.models import HTTPResponse
import netlib.tutils
from netlib.http import CONTENT_MISSING
from mitmproxy import dump, flow
from mitmproxy.proxy import Log
@ -38,7 +37,7 @@ def test_strfuncs():
flow.request.stickycookie = True
flow.client_conn = mock.MagicMock()
flow.client_conn.address.host = "foo"
flow.response = netlib.tutils.tresp(content=CONTENT_MISSING)
flow.response = netlib.tutils.tresp(content=None)
flow.response.is_replay = True
flow.response.status_code = 300
m.echo_flow(flow)
@ -104,10 +103,10 @@ class TestDumpMaster:
o = dump.Options(flow_detail=3)
m = dump.DumpMaster(None, o, outfile=cs)
f = tutils.tflow()
f.request.content = CONTENT_MISSING
f.request.content = None
m.handle_request(f)
f.response = HTTPResponse.wrap(netlib.tutils.tresp())
f.response.content = CONTENT_MISSING
f.response.content = None
m.handle_response(f)
assert "content missing" in cs.getvalue()

6
test/mitmproxy/test_flow.py
View File

@ -8,7 +8,7 @@ import mock
import netlib.utils
from netlib import odict
from netlib.http import CONTENT_MISSING, Headers
from netlib.http import Headers
from mitmproxy import filt, controller, tnetstring, flow
from mitmproxy.models import Error
from mitmproxy.models import Flow
@ -465,7 +465,7 @@ class TestFlow(object):
def test_replace_no_content(self):
f = tutils.tflow()
f.request.content = CONTENT_MISSING
f.request.content = None
assert f.replace("foo", "bar") == 0
def test_replace(self):
@ -751,7 +751,7 @@ class TestFlowMaster:
s = flow.State()
fm = flow.FlowMaster(None, s)
f = tutils.tflow(resp=True)
f.request.content = CONTENT_MISSING
f.request.content = None
assert "missing" in fm.replay_request(f)
f.intercepted = True

23
test/mitmproxy/test_flow_export.py
View File

@ -1,3 +1,4 @@
import json
from textwrap import dedent
import netlib.tutils
@ -178,6 +179,7 @@ class TestRawRequest():
""").strip()
assert flow_export.raw_request(flow) == result
class TestExportLocustCode():
def test_get(self):
@ -369,3 +371,24 @@ class TestExportLocustTask():
""".strip() + '\n'
assert flow_export.locust_task(flow) == result
class TestIsJson():
def test_empty(self):
assert flow_export.is_json(None, None) == False
def test_json_type(self):
headers = Headers(content_type="application/json")
assert flow_export.is_json(headers, "foobar") == False
def test_valid(self):
headers = Headers(content_type="application/foobar")
j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}')
assert j == False
def test_valid(self):
headers = Headers(content_type="application/json")
j = flow_export.is_json(headers, '{"name": "example", "email": "example@example.com"}')
assert isinstance(j, dict)

6
test/mitmproxy/test_server.py
View File

@ -8,7 +8,7 @@ from netlib.tcp import Address
import netlib.tutils
from netlib import tcp, http, socks
from netlib.certutils import SSLCert
from netlib.http import authentication, CONTENT_MISSING, http1
from netlib.http import authentication, http1
from netlib.tutils import raises
from pathod import pathoc, pathod
@ -281,7 +281,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin):
self.pathod("200:b@3k")
assert self.master.state.view[-1].response.stream
assert self.master.state.view[-1].response.content == CONTENT_MISSING
assert self.master.state.view[-1].response.content is None
self.master.set_stream_large_bodies(None)
def test_stream_modify(self):
@ -816,7 +816,7 @@ class MasterIncomplete(tservers.TestMaster):
def handle_request(self, f):
resp = HTTPResponse.wrap(netlib.tutils.tresp())
resp.content = CONTENT_MISSING
resp.content = None
f.reply(resp)

6
test/netlib/http/http1/test_assemble.py
View File

@ -1,6 +1,6 @@
from __future__ import absolute_import, print_function, division
from netlib.exceptions import HttpException
from netlib.http import CONTENT_MISSING, Headers
from netlib.http import Headers
from netlib.http.http1.assemble import (
assemble_request, assemble_request_head, assemble_response,
assemble_response_head, _assemble_request_line, _assemble_request_headers,
@ -20,7 +20,7 @@ def test_assemble_request():
)
with raises(HttpException):
assemble_request(treq(content=CONTENT_MISSING))
assemble_request(treq(content=None))
def test_assemble_request_head():
@ -41,7 +41,7 @@ def test_assemble_response():
)
with raises(HttpException):
assemble_response(tresp(content=CONTENT_MISSING))
assemble_response(tresp(content=None))
def test_assemble_response_head():