Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-26 10:16:27 +00:00
Code Issues Projects Releases Wiki Activity

mitmproxy: zap six

Browse Source
This commit is contained in:
Aldo Cortesi 2016-10-17 16:20:35 +13:00
parent d60ef617e3
commit 839813a84c
24 changed files with 97 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mitmproxy/builtins/serverplayback.py
View File

@ -1,5 +1,5 @@
from __future__ import absolute_import, print_function, division
from six.moves import urllib
import urllib
import hashlib
from netlib import strutils

7
mitmproxy/builtins/stickycookie.py
View File

@ -1,5 +1,6 @@
import collections
from six.moves import http_cookiejar
from http import cookiejar
from netlib.http import cookies
from mitmproxy import exceptions
@ -20,9 +21,9 @@ def ckey(attrs, f):
def domain_match(a, b):
if http_cookiejar.domain_match(a, b):
if cookiejar.domain_match(a, b):
return True
elif http_cookiejar.domain_match(a, b.strip(".")):
elif cookiejar.domain_match(a, b.strip(".")):
return True
return False

4
mitmproxy/console/grideditor/base.py
View File

@ -2,7 +2,6 @@ from __future__ import absolute_import, print_function, division
import abc
import copy
import six
import urwid
from mitmproxy.console import common
from mitmproxy.console import signals
@ -24,8 +23,7 @@ FOOTER_EDITING = [
]
@six.add_metaclass(abc.ABCMeta)
class Column(object):
class Column(object, metaclass=abc.ABCMeta):
subeditor = None
def __init__(self, heading):

10
mitmproxy/console/master.py
View File

@ -13,7 +13,6 @@ import tempfile
import traceback
import weakref
import six
import urwid
from typing import Optional # noqa
@ -357,13 +356,8 @@ class ConsoleMaster(flow.FlowMaster):
def spawn_editor(self, data):
text = not isinstance(data, bytes)
fd, name = tempfile.mkstemp('', "mproxy", text=text)
if six.PY2:
os.close(fd)
with open(name, "w" if text else "wb") as f:
f.write(data)
else:
with open(fd, "w" if text else "wb") as f:
f.write(data)
with open(fd, "w" if text else "wb") as f:
f.write(data)
# if no EDITOR is set, assume 'vi'
c = os.environ.get("EDITOR") or "vi"
cmd = shlex.split(c)

16
mitmproxy/contentviews.py
View File

@ -16,24 +16,26 @@ from __future__ import absolute_import, print_function, division
import cssutils
import datetime
import html2text
import jsbeautifier
import json
import logging
import lxml.etree
import lxml.html
import subprocess
import traceback
import io
from typing import Mapping # noqa
import lxml.etree
import lxml.html
from PIL import ExifTags
from PIL import Image
import html2text
import jsbeautifier
from mitmproxy import exceptions
from mitmproxy.contrib.wbxml import ASCommandResponse
from netlib import http
from netlib import multidict
from netlib import strutils
from netlib.http import url
from six import BytesIO
from typing import Mapping # noqa
try:
import pyamf
@ -422,7 +424,7 @@ class ViewImage(View):
def __call__(self, data, **metadata):
try:
img = Image.open(BytesIO(data))
img = Image.open(io.BytesIO(data))
except IOError:
return None
parts = [

11
mitmproxy/contrib/tls/utils.py
View File

@ -6,15 +6,12 @@ from __future__ import absolute_import, division, print_function
import construct
import six
class _UBInt24(construct.Adapter):
def _encode(self, obj, context):
return (
six.int2byte((obj & 0xFF0000) >> 16) +
six.int2byte((obj & 0x00FF00) >> 8) +
six.int2byte(obj & 0x0000FF)
return bytes(
(obj & 0xFF0000) >> 16,
(obj & 0x00FF00) >> 8,
obj & 0x0000FF
)
def _decode(self, obj, context):

7
mitmproxy/contrib/tnetstring.py
View File

@ -41,7 +41,6 @@ all other strings are returned as plain bytes.
"""
import collections
import six
from typing import io, Union, Tuple # noqa
TSerializable = Union[None, bool, int, float, bytes, list, tuple, dict]
@ -96,7 +95,7 @@ def _rdumpq(q, size, value):
elif value is False:
write(b'5:false!')
return size + 8
elif isinstance(value, six.integer_types):
elif isinstance(value, int):
data = str(value).encode()
ldata = len(data)
span = str(ldata).encode()
@ -191,16 +190,12 @@ def load(file_handle):
def parse(data_type, data):
# type: (int, bytes) -> TSerializable
if six.PY2:
data_type = ord(data_type)
if data_type == ord(b','):
return data
if data_type == ord(b';'):
return data.decode("utf8")
if data_type == ord(b'#'):
try:
if six.PY2:
return long(data)
return int(data)
except ValueError:
raise ValueError("not a tnetstring: invalid integer literal: {}".format(data))

21
mitmproxy/contrib/wbxml/ASWBXMLByteQueue.py
View File

@ -5,7 +5,7 @@
Inspired by EAS Inspector for Fiddler
https://easinspectorforfiddler.codeplex.com
----- The MIT License (MIT) -----
----- The MIT License (MIT) -----
Filename: ASWBXMLByteQueue.py
Copyright (c) 2014, David P. Shaw
@ -27,25 +27,25 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
'''
from six.moves.queue import Queue
from queue import Queue
import logging
class ASWBXMLByteQueue(Queue):
def __init__(self, wbxmlBytes):
self.bytesDequeued = 0
self.bytesEnqueued = 0
Queue.__init__(self)
for byte in wbxmlBytes:
self.put(ord(byte))
self.bytesEnqueued += 1
logging.debug("Array byte count: %d, enqueued: %d" % (self.qsize(), self.bytesEnqueued))
"""
Created to debug the dequeueing of bytes
"""
@ -54,18 +54,18 @@ class ASWBXMLByteQueue(Queue):
self.bytesDequeued += 1
logging.debug("Dequeued byte 0x{0:X} ({1} total)".format(singleByte, self.bytesDequeued))
return singleByte
"""
Return true if the continuation bit is set in the byte
"""
def checkContinuationBit(self, byteval):
continuationBitmask = 0x80
return (continuationBitmask & byteval) != 0
def dequeueMultibyteInt(self):
iReturn = 0
singleByte = 0xFF
while True:
iReturn <<= 7
if (self.qsize() == 0):
@ -100,4 +100,3 @@ class ASWBXMLByteQueue(Queue):
break
return strReturn

3
mitmproxy/controller.py
View File

@ -3,8 +3,7 @@ from __future__ import absolute_import, print_function, division
import functools
import threading
import contextlib
from six.moves import queue
import queue
from mitmproxy import addons
from mitmproxy import options

18
mitmproxy/flow/export.py
View File

@ -2,21 +2,15 @@ from __future__ import absolute_import, print_function, division
import json
import re
from textwrap import dedent
import six
from six.moves import urllib
import textwrap
import urllib
import netlib.http
def _native(s):
if six.PY2:
if isinstance(s, str):
return s.encode()
else:
if isinstance(s, bytes):
return s.decode()
if isinstance(s, bytes):
return s.decode()
return s
@ -49,7 +43,7 @@ def curl_command(flow):
def python_code(flow):
code = dedent("""
code = textwrap.dedent("""
import requests
url = '{url}'
@ -110,7 +104,7 @@ def is_json(headers, content):
def locust_code(flow):
code = dedent("""
code = textwrap.dedent("""
from locust import HttpLocust, TaskSet, task
class UserBehavior(TaskSet):

27
mitmproxy/flow/io_compat.py
View File

@ -3,7 +3,6 @@ This module handles the import of mitmproxy flows generated by old versions.
"""
from __future__ import absolute_import, print_function, division
import six
from typing import Any # noqa
from netlib import version, strutils
@ -93,23 +92,21 @@ def _convert_dict_vals(o, values_to_convert):
def convert_unicode(data):
# type: (dict) -> dict
"""
The Python 2 version of mitmproxy serializes everything as bytes.
This method converts between Python 3 and Python 2 dumpfiles.
"""
if not six.PY2:
data = _convert_dict_keys(data)
data = _convert_dict_vals(
data, {
"type": True,
"id": True,
"request": {
"first_line_format": True
},
"error": {
"msg": True
}
data = _convert_dict_keys(data)
data = _convert_dict_vals(
data, {
"type": True,
"id": True,
"request": {
"first_line_format": True
},
"error": {
"msg": True
}
)
}
)
return data

7
mitmproxy/flow/state.py
View File

@ -2,15 +2,13 @@ from __future__ import absolute_import, print_function, division
from abc import abstractmethod, ABCMeta
import six
from typing import List # noqa
from mitmproxy import flowfilter
from mitmproxy import models # noqa
@six.add_metaclass(ABCMeta)
class FlowList(object):
class FlowList(metaclass=ABCMeta):
def __init__(self):
self._list = [] # type: List[models.Flow]
@ -26,9 +24,6 @@ class FlowList(object):
def __bool__(self):
return bool(self._list)
if six.PY2:
__nonzero__ = __bool__
def __len__(self):
return len(self._list)

8
mitmproxy/main.py
View File

@ -4,8 +4,6 @@ import os
import signal
import sys
from six.moves import _thread # PY3: We only need _thread.error, which is an alias of RuntimeError in 3.3+
from mitmproxy import cmdline
from mitmproxy import exceptions
from mitmproxy.proxy import config
@ -78,7 +76,7 @@ def mitmproxy(args=None): # pragma: no cover
sys.exit(1)
try:
m.run()
except (KeyboardInterrupt, _thread.error):
except (KeyboardInterrupt, RuntimeError):
pass
@ -109,7 +107,7 @@ def mitmdump(args=None): # pragma: no cover
except (dump.DumpError, exceptions.OptionsError) as e:
print("mitmdump: %s" % e, file=sys.stderr)
sys.exit(1)
except (KeyboardInterrupt, _thread.error):
except (KeyboardInterrupt, RuntimeError):
pass
if master is None or master.has_errored:
print("mitmdump: errors occurred during run", file=sys.stderr)
@ -142,5 +140,5 @@ def mitmweb(args=None): # pragma: no cover
sys.exit(1)
try:
m.run()
except (KeyboardInterrupt, _thread.error):
except (KeyboardInterrupt, RuntimeError):
pass

8
mitmproxy/models/connections.py
View File

@ -4,8 +4,6 @@ import time
import copy
import os
import six
from mitmproxy import stateobject
from netlib import certutils
from netlib import tcp
@ -47,9 +45,6 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
def __bool__(self):
return bool(self.connection) and not self.finished
if six.PY2:
__nonzero__ = __bool__
def __repr__(self):
return "<ClientConnection: {ssl}{address}>".format(
ssl="[ssl] " if self.ssl_established else "",
@ -136,9 +131,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
def __bool__(self):
return bool(self.connection) and not self.finished
if six.PY2:
__nonzero__ = __bool__
def __repr__(self):
if self.ssl_established and self.sni:
ssl = "[ssl: {0}] ".format(self.sni)

4
mitmproxy/platform/windows.py
View File

@ -10,8 +10,8 @@ import time
import configargparse
from pydivert import enum
from pydivert import windivert
from six.moves import cPickle as pickle
from six.moves import socketserver
import pickle
import socketserver
PROXY_API_PORT = 8085

16
mitmproxy/protocol/base.py
View File

@ -1,9 +1,5 @@
from __future__ import absolute_import, print_function, division
import sys
import six
import netlib.exceptions
from mitmproxy import exceptions
from mitmproxy import models
@ -184,12 +180,8 @@ class ServerConnectionMixin(object):
try:
self.server_conn.connect()
except netlib.exceptions.TcpException as e:
six.reraise(
exceptions.ProtocolException,
exceptions.ProtocolException(
"Server connection to {} failed: {}".format(
repr(self.server_conn.address), str(e)
)
),
sys.exc_info()[2]
raise exceptions.ProtocolException(
"Server connection to {} failed: {}".format(
repr(self.server_conn.address), str(e)
)
)

23
mitmproxy/protocol/http.py
View File

@ -2,8 +2,6 @@ from __future__ import absolute_import, print_function, division
import h2.exceptions
import netlib.exceptions
import six
import sys
import time
import traceback
from mitmproxy import exceptions
@ -83,9 +81,6 @@ class ConnectServerConnection(object):
def __bool__(self):
return bool(self.via)
if six.PY2:
__nonzero__ = __bool__
class UpstreamConnectLayer(base.Layer):
@ -159,12 +154,8 @@ class HttpLayer(base.Layer):
# We optimistically guess there might be an HTTP client on the
# other end
self.send_error_response(400, repr(e))
six.reraise(
exceptions.ProtocolException,
exceptions.ProtocolException(
"HTTP protocol error in client request: {}".format(e)
),
sys.exc_info()[2]
raise exceptions.ProtocolException(
"HTTP protocol error in client request: {}".format(e)
)
self.log("request", "debug", [repr(request)])
@ -193,10 +184,7 @@ class HttpLayer(base.Layer):
# update host header in reverse proxy mode
if self.config.options.mode == "reverse":
if six.PY2:
flow.request.headers["Host"] = self.config.upstream_server.address.host.encode()
else:
flow.request.headers["Host"] = self.config.upstream_server.address.host
flow.request.headers["Host"] = self.config.upstream_server.address.host
# set upstream auth
if self.mode == "upstream" and self.config.upstream_auth is not None:
@ -244,8 +232,9 @@ class HttpLayer(base.Layer):
self.channel.ask("error", flow)
return
else:
six.reraise(exceptions.ProtocolException, exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2])
raise exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)
)
finally:
if flow:
flow.live = False

5
mitmproxy/protocol/http2.py
View File

@ -6,10 +6,9 @@ import traceback
import functools
import h2.exceptions
import six
from h2 import connection
from h2 import events
from six.moves import queue
import queue
import netlib.exceptions
from mitmproxy import exceptions
@ -208,7 +207,7 @@ class Http2Layer(base.Layer):
return True
def _handle_remote_settings_changed(self, event, other_conn):
new_settings = dict([(key, cs.new_value) for (key, cs) in six.iteritems(event.changed_settings)])
new_settings = dict([(key, cs.new_value) for (key, cs) in event.changed_settings.items()])
other_conn.h2.safe_update_settings(new_settings)
return True

57
mitmproxy/protocol/tls.py
View File

@ -1,10 +1,8 @@
from __future__ import absolute_import, print_function, division
import struct
import sys
import construct
import six
import netlib.exceptions
from mitmproxy import exceptions
@ -214,20 +212,12 @@ def is_tls_record_magic(d):
# TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2
# http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello
if six.PY2:
return (
len(d) == 3 and
d[0] == '\x16' and
d[1] == '\x03' and
d[2] in ('\x00', '\x01', '\x02', '\x03')
)
else:
return (
len(d) == 3 and
d[0] == 0x16 and
d[1] == 0x03 and
0x0 <= d[2] <= 0x03
)
return (
len(d) == 3 and
d[0] == 0x16 and
d[1] == 0x03 and
0x0 <= d[2] <= 0x03
)
def get_client_hello(client_conn):
@ -467,7 +457,7 @@ class TlsLayer(base.Layer):
self._establish_tls_with_client()
except:
pass
six.reraise(*sys.exc_info())
raise
self._establish_tls_with_client()
@ -497,15 +487,11 @@ class TlsLayer(base.Layer):
# raises ann error.
self.client_conn.rfile.peek(1)
except netlib.exceptions.TlsException as e:
six.reraise(
exceptions.ClientHandshakeException,
exceptions.ClientHandshakeException(
"Cannot establish TLS with client (sni: {sni}): {e}".format(
sni=self._client_hello.sni, e=repr(e)
),
self._client_hello.sni or repr(self.server_conn.address)
raise exceptions.ClientHandshakeException(
"Cannot establish TLS with client (sni: {sni}): {e}".format(
sni=self._client_hello.sni, e=repr(e)
),
sys.exc_info()[2]
self._client_hello.sni or repr(self.server_conn.address)
)
def _establish_tls_with_server(self):
@ -545,21 +531,14 @@ class TlsLayer(base.Layer):
self.log(str(tls_cert_err), "warn")
self.log("Ignoring server verification error, continuing with connection", "warn")
except netlib.exceptions.InvalidCertificateException as e:
six.reraise(
exceptions.InvalidServerCertificate,
exceptions.InvalidServerCertificate(str(e)),
sys.exc_info()[2]
)
raise exceptions.InvalidServerCertificate(str(e))
except netlib.exceptions.TlsException as e:
six.reraise(
exceptions.TlsProtocolException,
exceptions.TlsProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format(
address=repr(self.server_conn.address),
sni=self.server_sni,
e=repr(e),
)),
sys.exc_info()[2]
)
raise exceptions.TlsProtocolException(
"Cannot establish TLS with {address} (sni: {sni}): {e}".format(
address=repr(self.server_conn.address),
sni=self.server_sni,
e=repr(e),
))
proto = self.alpn_for_client_connection.decode() if self.alpn_for_client_connection else '-'
self.log("ALPN selected by server: {}".format(proto), "debug")

4
mitmproxy/proxy/config.py
View File

@ -6,7 +6,6 @@ import os
import re
from netlib import strutils
import six
from OpenSSL import SSL, crypto
from mitmproxy import exceptions
@ -38,9 +37,6 @@ class HostMatcher(object):
def __bool__(self):
return bool(self.patterns)
if six.PY2:
__nonzero__ = __bool__
ServerSpec = collections.namedtuple("ServerSpec", "scheme address")

8
mitmproxy/proxy/root_context.py
View File

@ -1,9 +1,5 @@
from __future__ import absolute_import, print_function, division
import sys
import six
import netlib.exceptions
from mitmproxy import controller
from mitmproxy import exceptions
@ -50,7 +46,7 @@ class RootContext(object):
try:
d = top_layer.client_conn.rfile.peek(3)
except netlib.exceptions.TcpException as e:
six.reraise(exceptions.ProtocolException, exceptions.ProtocolException(str(e)), sys.exc_info()[2])
raise exceptions.ProtocolException(str(e))
client_tls = protocol.is_tls_record_magic(d)
# 1. check for --ignore
@ -101,7 +97,7 @@ class RootContext(object):
is_ascii = (
len(d) == 3 and
# expect A-Za-z
all(65 <= x <= 90 or 97 <= x <= 122 for x in six.iterbytes(d))
all(65 <= x <= 90 or 97 <= x <= 122 for x in d)
)
if self.config.options.rawtcp and not is_ascii:
return protocol.RawTCPLayer(top_layer)

8
mitmproxy/proxy/server.py
View File

@ -4,8 +4,6 @@ import socket
import sys
import traceback
import six
import netlib.exceptions
from mitmproxy import exceptions
from mitmproxy import models
@ -46,10 +44,8 @@ class ProxyServer(tcp.TCPServer):
(config.options.listen_host, config.options.listen_port)
)
except socket.error as e:
six.reraise(
exceptions.ServerException,
exceptions.ServerException('Error starting proxy server: ' + repr(e)),
sys.exc_info()[2]
raise exceptions.ServerException(
'Error starting proxy server: ' + repr(e)
)
self.channel = None

5
mitmproxy/stateobject.py
View File

@ -1,6 +1,5 @@
from __future__ import absolute_import, print_function, division
import six
from typing import Any
from typing import List
@ -34,7 +33,7 @@ class StateObject(netlib.basetypes.Serializable):
Retrieve object state.
"""
state = {}
for attr, cls in six.iteritems(self._stateobject_attributes):
for attr, cls in self._stateobject_attributes.items():
val = getattr(self, attr)
if val is None:
state[attr] = None
@ -51,7 +50,7 @@ class StateObject(netlib.basetypes.Serializable):
Load object state from data returned by a get_state call.
"""
state = state.copy()
for attr, cls in six.iteritems(self._stateobject_attributes):
for attr, cls in self._stateobject_attributes.items():
val = state.pop(attr)
if val is None:
setattr(self, attr, val)

9
mitmproxy/web/app.py
View File

@ -8,7 +8,6 @@ import re
import hashlib
import six
import tornado.websocket
import tornado.web
from io import BytesIO
@ -242,10 +241,10 @@ class FlowHandler(RequestHandler):
def put(self, flow_id):
flow = self.flow
flow.backup()
for a, b in six.iteritems(self.json):
for a, b in self.json.items():
if a == "request":
request = flow.request
for k, v in six.iteritems(b):
for k, v in b.items():
if k in ["method", "scheme", "host", "path", "http_version"]:
setattr(request, k, str(v))
elif k == "port":
@ -259,7 +258,7 @@ class FlowHandler(RequestHandler):
elif a == "response":
response = flow.response
for k, v in six.iteritems(b):
for k, v in b.items():
if k == "msg":
response.msg = str(v)
elif k == "code":
@ -387,7 +386,7 @@ class Settings(RequestHandler):
def put(self):
update = {}
for k, v in six.iteritems(self.json):
for k, v in self.json.items():
if k == "intercept":
self.master.options.intercept = v
update[k] = v