Merge branch 'master' into solidcore

This commit is contained in:
Aldo Cortesi 2016-05-29 11:14:46 +12:00
commit ed415877d4
57 changed files with 124 additions and 117 deletions

1
.gitignore vendored
View File

@ -16,3 +16,4 @@ build/
node_modules
bower_components
*.map
sslkeylogfile.log

View File

@ -22,9 +22,9 @@ matrix:
git:
depth: 9999999
- python: 3.5
env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py"
env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py ./test/pathod/test_log.py"
- python: 3.5
env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py" NO_ALPN=1
env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py ./test/pathod/test_log.py" NO_ALPN=1
- python: 2.7
env: DOCS=1
script: 'cd docs && make html'

View File

@ -12,6 +12,7 @@ class ControlError(Exception):
class Master(object):
"""
The master handles mitmproxy's main event loop.
"""
@ -60,6 +61,7 @@ class Master(object):
class ServerMaster(Master):
"""
The ServerMaster adds server thread support to the master.
"""
@ -86,6 +88,7 @@ class ServerMaster(Master):
class ServerThread(threading.Thread):
def __init__(self, server):
self.server = server
super(ServerThread, self).__init__()
@ -97,6 +100,7 @@ class ServerThread(threading.Thread):
class Channel(object):
"""
The only way for the proxy server to communicate with the master
is to use the channel it has been given.
@ -137,6 +141,7 @@ class Channel(object):
class DummyReply(object):
"""
A reply object that does nothing. Useful when we need an object to seem
like it has a channel, and during testing.
@ -187,6 +192,7 @@ def handler(f):
class Reply(object):
"""
Messages sent through a channel are decorated with a "reply" attribute.
This object is used to respond to the message through the return

View File

@ -13,6 +13,7 @@ import sys
class ProxyException(Exception):
"""
Base class for all exceptions thrown by mitmproxy.
"""
@ -22,6 +23,7 @@ class ProxyException(Exception):
class Kill(ProxyException):
"""
Signal that both client and server connection(s) should be killed immediately.
"""
@ -37,6 +39,7 @@ class TlsProtocolException(ProtocolException):
class ClientHandshakeException(TlsProtocolException):
def __init__(self, message, server):
super(ClientHandshakeException, self).__init__(message)
self.server = server
@ -67,6 +70,7 @@ class ReplayException(ProxyException):
class ScriptException(ProxyException):
@classmethod
def from_exception_context(cls, cut_tb=1):
"""

View File

@ -38,6 +38,7 @@ import pyparsing as pp
class _Token(object):
def dump(self, indent=0, fp=sys.stdout):
print("{spacing}{name}{expr}".format(
spacing="\t" * indent,

View File

@ -8,13 +8,13 @@ import hashlib
import sys
import six
from six.moves import http_cookies, http_cookiejar, urllib
from six.moves import http_cookiejar
from six.moves import urllib
import os
import re
from typing import List, Optional, Set
from netlib import wsgi, odict
from netlib import wsgi
from netlib.exceptions import HttpException
from netlib.http import Headers, http1, cookies
from netlib.utils import clean_bin

View File

@ -5,7 +5,9 @@ import netlib.http
from netlib.utils import parse_content_type
import re
from six.moves.urllib.parse import urlparse, quote, quote_plus
from six.moves.urllib.parse import quote
from six.moves.urllib.parse import quote_plus
def curl_command(flow):
data = "curl "
@ -123,13 +125,12 @@ def locust_code(flow):
max_wait = 3000
""").strip()
components = map(lambda x: quote(x, safe=""), flow.request.path_components)
file_name = "_".join(components)
name = re.sub('\W|^(?=\d)', '_', file_name)
url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components)
if name == "" or name is None:
new_name = "_".join([str(flow.request.host) , str(flow.request.timestamp_start)])
new_name = "_".join([str(flow.request.host), str(flow.request.timestamp_start)])
name = re.sub('\W|^(?=\d)', '_', new_name)
args = ""
headers = ""

View File

@ -10,6 +10,7 @@ from .. import stateobject, utils
class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
"""
A client connection
@ -21,6 +22,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
timestamp_ssl_setup: TLS established timestamp
timestamp_end: Connection end timestamp
"""
def __init__(self, client_connection, address, server):
# Eventually, this object is restored from state. We don't have a
# connection then.
@ -101,6 +103,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
class ServerConnection(tcp.TCPClient, stateobject.StateObject):
"""
A server connection
@ -117,6 +120,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
timestamp_ssl_setup: TLS established timestamp
timestamp_end: Connection end timestamp
"""
def __init__(self, address, source_address=None):
tcp.TCPClient.__init__(self, address, source_address)

View File

@ -72,9 +72,9 @@ class HTTPRequest(MessageMixin, Request):
def get_state(self):
state = super(HTTPRequest, self).get_state()
state.update(
stickycookie = self.stickycookie,
stickyauth = self.stickyauth,
is_replay = self.is_replay,
stickycookie=self.stickycookie,
stickyauth=self.stickyauth,
is_replay=self.is_replay,
)
return state
@ -109,6 +109,7 @@ class HTTPRequest(MessageMixin, Request):
class HTTPResponse(MessageMixin, Response):
"""
A mitmproxy HTTP response.
This is a very thin wrapper on top of :py:class:`netlib.http.Response` and
@ -124,7 +125,7 @@ class HTTPResponse(MessageMixin, Response):
content,
timestamp_start=None,
timestamp_end=None,
is_replay = False
is_replay=False
):
Response.__init__(
self,

View File

@ -6,6 +6,7 @@ from .flow import Flow
class TCPMessage(Serializable):
def __init__(self, from_client, content, timestamp=None):
self.content = content
self.from_client = from_client
@ -33,6 +34,7 @@ class TCPMessage(Serializable):
class TCPFlow(Flow):
"""
A TCPFlow is a simplified representation of a TCP session.
"""

View File

@ -62,7 +62,7 @@ class RequestReplayThread(threading.Thread):
)
r.first_line_format = "relative"
else:
r.first_line_format= "absolute"
r.first_line_format = "absolute"
else:
server_address = (r.host, r.port)
server = ServerConnection(server_address, (self.config.host, 0))

View File

@ -1,14 +1,10 @@
from __future__ import (absolute_import, print_function, division)
import socket
import six
import sys
from OpenSSL import SSL
from netlib.exceptions import TcpException
from netlib.tcp import ssl_read_select
from netlib.utils import clean_bin
from ..exceptions import ProtocolException
from ..models import Error
from ..models.tcp import TCPFlow, TCPMessage

View File

@ -312,6 +312,7 @@ class TlsClientHello(object):
class TlsLayer(Layer):
"""
The TLS layer implements transparent TLS connections.
@ -482,7 +483,7 @@ class TlsLayer(Layer):
dhparams=self.config.certstore.dhparams,
chain_file=chain_file,
alpn_select_callback=self.__alpn_select_callback,
extra_chain_certs = extra_certs,
extra_chain_certs=extra_certs,
)
# Some TLS clients will not fail the handshake,
# but will immediately throw an "unexpected eof" error on the first read.

View File

@ -58,7 +58,7 @@ class ProxyConfig:
body_size_limit=None,
mode="regular",
upstream_server=None,
upstream_auth = None,
upstream_auth=None,
authenticator=None,
ignore_hosts=tuple(),
tcp_hosts=tuple(),

View File

@ -6,7 +6,6 @@ by the mitmproxy-specific ScriptContext.
# Do not import __future__ here, this would apply transitively to the inline scripts.
import os
import shlex
import traceback
import sys
import six

View File

@ -68,6 +68,7 @@ like so::
"""
import six
from collections import deque
__ver_major__ = 0
__ver_minor__ = 2
@ -77,9 +78,6 @@ __version__ = "%d.%d.%d%s" % (
__ver_major__, __ver_minor__, __ver_patch__, __ver_sub__)
from collections import deque
def dumps(value, encoding=None):
"""dumps(object,encoding=None) -> string

View File

@ -3,7 +3,6 @@ import re
from email.utils import parsedate_tz, formatdate, mktime_tz
from netlib.multidict import ImmutableMultiDict
from .. import odict
"""
A flexible module for cookie parsing and manipulation.
@ -28,6 +27,7 @@ variants. Serialization follows RFC6265.
# TODO: Disallow LHS-only Cookie values
def _read_until(s, start, term):
"""
Read until one of the characters in term is reached.

View File

@ -14,12 +14,18 @@ from ..utils import always_bytes
# See also: http://lucumr.pocoo.org/2013/7/2/the-updated-guide-to-unicode/
if six.PY2: # pragma: no cover
_native = lambda x: x
_always_bytes = lambda x: x
def _native(x):
return x
def _always_bytes(x):
return x
else:
# While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
_native = lambda x: x.decode("utf-8", "surrogateescape")
_always_bytes = lambda x: always_bytes(x, "utf-8", "surrogateescape")
def _native(x):
return x.decode("utf-8", "surrogateescape")
def _always_bytes(x):
return always_bytes(x, "utf-8", "surrogateescape")
class Headers(MultiDict):

View File

@ -1,9 +1,9 @@
from __future__ import absolute_import, print_function, division
from ... import utils
import itertools
from ...exceptions import HttpException
def assemble_request(request):
if request.content is None:
raise HttpException("Cannot assemble flow with missing content")

View File

@ -4,17 +4,23 @@ import warnings
import six
from ..multidict import MultiDict
from .headers import Headers
from .. import encoding, utils
from ..utils import always_bytes
if six.PY2: # pragma: no cover
_native = lambda x: x
_always_bytes = lambda x: x
def _native(x):
return x
def _always_bytes(x):
return x
else:
# While the HTTP head _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
_native = lambda x: x.decode("utf-8", "surrogateescape")
_always_bytes = lambda x: utils.always_bytes(x, "utf-8", "surrogateescape")
# While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
def _native(x):
return x.decode("utf-8", "surrogateescape")
def _always_bytes(x):
return always_bytes(x, "utf-8", "surrogateescape")
class MessageData(utils.Serializable):

View File

@ -1,14 +1,12 @@
from __future__ import absolute_import, print_function, division
import re
import warnings
import six
from six.moves import urllib
from netlib import utils
from netlib.http import cookies
from netlib.odict import ODict
from .. import encoding
from ..multidict import MultiDictView
from .headers import Headers

View File

@ -2,7 +2,6 @@ from __future__ import absolute_import, print_function, division
from abc import ABCMeta, abstractmethod
from typing import Tuple, TypeVar
try:
from collections.abc import MutableMapping

View File

@ -71,6 +71,7 @@ sslversion_choices = {
"TLSv1_2": (SSL.TLSv1_2_METHOD, SSL_BASIC_OPTIONS),
}
class SSLKeyLogger(object):
def __init__(self, filename):

View File

@ -91,8 +91,7 @@ class RaisesContext(object):
test_data = utils.Data(__name__)
# FIXME: Temporary workaround during repo merge.
import os
test_data.dirname = os.path.join(test_data.dirname,"..","test","netlib")
test_data.dirname = os.path.join(test_data.dirname, "..", "test", "netlib")
def treq(**kwargs):

View File

@ -10,7 +10,6 @@ import os.path
import six
import OpenSSL
from . import version
PYOPENSSL_MIN_VERSION = (0, 15)

View File

@ -14,7 +14,7 @@ from netlib import utils
MAX_16_BIT_INT = (1 << 16)
MAX_64_BIT_INT = (1 << 64)
DEFAULT=object()
DEFAULT = object()
OPCODE = utils.BiDi(
CONTINUE=0x00,

View File

@ -1,24 +1,23 @@
"""
Colleciton of utility functions that implement small portions of the RFC6455
WebSockets Protocol Useful for building WebSocket clients and servers.
Emphassis is on readabilty, simplicity and modularity, not performance or
completeness
This is a work in progress and does not yet contain all the utilites need to
create fully complient client/servers #
Spec: https://tools.ietf.org/html/rfc6455
# Colleciton of utility functions that implement small portions of the RFC6455
# WebSockets Protocol Useful for building WebSocket clients and servers.
#
# Emphassis is on readabilty, simplicity and modularity, not performance or
# completeness
#
# This is a work in progress and does not yet contain all the utilites need to
# create fully complient client/servers #
# Spec: https://tools.ietf.org/html/rfc6455
The magic sha that websocket servers must know to prove they understand
RFC6455
"""
# The magic sha that websocket servers must know to prove they understand
# RFC6455
from __future__ import absolute_import
import base64
import hashlib
import os
import binascii
import six
from ..http import Headers
@ -95,21 +94,18 @@ class WebsocketsProtocol(object):
upgrade="websocket"
)
@classmethod
def check_client_handshake(self, headers):
if headers.get("upgrade") != "websocket":
return
return headers.get("sec-websocket-key")
@classmethod
def check_server_handshake(self, headers):
if headers.get("upgrade") != "websocket":
return
return headers.get("sec-websocket-accept")
@classmethod
def create_server_nonce(self, client_nonce):
return base64.b64encode(hashlib.sha1(client_nonce + websockets_magic).digest())

View File

@ -1,15 +1,15 @@
from __future__ import (absolute_import, print_function, division)
from io import BytesIO, StringIO
import urllib
import time
import traceback
import six
from io import BytesIO
from six.moves import urllib
from netlib.utils import always_bytes, native
from . import http, tcp
class ClientConn(object):
def __init__(self, address):
@ -140,7 +140,7 @@ class WSGIAdaptor(object):
elif state["status"]:
raise AssertionError('Response already started')
state["status"] = status
state["headers"] = http.Headers([[always_bytes(k), always_bytes(v)] for k,v in headers])
state["headers"] = http.Headers([[always_bytes(k), always_bytes(v)] for k, v in headers])
if exc_info:
self.error_page(soc, state["headers_sent"], traceback.format_tb(exc_info[2]))
state["headers_sent"] = True
@ -154,7 +154,7 @@ class WSGIAdaptor(object):
write(i)
if not state["headers_sent"]:
write(b"")
except Exception as e:
except Exception:
try:
s = traceback.format_exc()
errs.write(s.encode("utf-8", "replace"))

View File

@ -11,6 +11,7 @@ from . import base, exceptions, actions, message
# instead of duplicating the HTTP on-the-wire representation here.
# see http2 language for an example
class WS(base.CaselessLiteral):
TOK = "ws"

View File

@ -27,6 +27,7 @@ from . import base, message
h2f:42:DATA:END_STREAM,PADDED:0x1234567:'content body payload'
"""
def get_header(val, headers):
"""
Header keys may be Values, so we have to "generate" them as we try the
@ -48,6 +49,7 @@ class _HeaderMixin(object):
self.value.get_generator(settings),
)
class _HTTP2Message(message.Message):
@property
def actions(self):
@ -287,13 +289,10 @@ class Request(_HTTP2Message):
def spec(self):
return ":".join([i.spec() for i in self.tokens])
def make_error_response(reason, body=None):
tokens = [
StatusCode("800"),
Body(base.TokValueLiteral("pathod error: " + (body or reason))),
]
return Response(tokens)
# class Frame(message.Message):
# pass

View File

@ -1,4 +1,3 @@
import os
import random
import string
import netlib.websockets

View File

@ -1,6 +1,5 @@
import time
from netlib.exceptions import TcpDisconnect
import netlib.tcp
BLOCKSIZE = 1024
# It's not clear what the upper limit for time.sleep is. It's lower than the

View File

@ -41,7 +41,7 @@ class SSLInfo(object):
"Cipher: %s, %s bit, %s" % self.cipher,
"SSL certificate chain:"
]
for n,i in enumerate(self.certchain):
for n, i in enumerate(self.certchain):
parts.append(" Certificate [%s]" % n)
parts.append("\tSubject: ")
for cn in i.get_subject().get_components():
@ -72,7 +72,6 @@ class SSLInfo(object):
return "\n".join(parts)
class WebsocketFrameReader(threading.Thread):
def __init__(

View File

@ -112,7 +112,6 @@ class PathodHandler(tcp.BaseHandler):
return None, response_log
return self.handle_http_request, response_log
def handle_http_request(self, logger):
"""
Returns a (handler, log) tuple.

View File

@ -49,12 +49,12 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
help="""
URL path specifying prefix for URL crafting
commands. (%s)
"""%pathod.DEFAULT_CRAFT_ANCHOR
""" % pathod.DEFAULT_CRAFT_ANCHOR
)
parser.add_argument(
"--confdir",
action="store", type = str, dest="confdir", default='~/.mitmproxy',
help = "Configuration directory. (~/.mitmproxy)"
action="store", type=str, dest="confdir", default='~/.mitmproxy',
help="Configuration directory. (~/.mitmproxy)"
)
parser.add_argument(
"-d", dest='staticdir', default=None, type=str,
@ -117,8 +117,8 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
)
group.add_argument(
"--cert", dest='ssl_certs', default=[], type=str,
metavar = "SPEC", action="append",
help = """
metavar="SPEC", action="append",
help="""
Add an SSL certificate. SPEC is of the form "[domain=]path". The domain
may include a wildcard, and is equal to "*" if not specified. The file
at path is a certificate in PEM format. If a private key is included in
@ -177,7 +177,6 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
help="Output all received & sent HTTP/2 frames"
)
args = parser.parse_args(argv[1:])
args.ssl_version, args.ssl_options = tcp.sslversion_choices[args.ssl_version]

View File

@ -1,6 +1,6 @@
from netlib import tcp, wsgi
from netlib.exceptions import HttpReadDisconnect, TlsException
from netlib.http import http1, Request
from netlib import wsgi
from netlib.exceptions import TlsException
from netlib.http import http1
from .. import version, language

View File

@ -1,5 +1,6 @@
from netlib.http import http2
from .. import version, app, language, utils, log
from .. import language
class HTTP2Protocol:

View File

@ -3,7 +3,7 @@ import json
import os
from contextlib import contextmanager
from mitmproxy import utils, script
from mitmproxy import script
from mitmproxy.proxy import config
import netlib.utils
from netlib import tutils as netutils

View File

@ -4,7 +4,6 @@ from six.moves import cStringIO as StringIO
import mock
import netlib.utils
from netlib import odict
from netlib.http import Headers
from mitmproxy import filt, controller, tnetstring, flow
from mitmproxy.exceptions import FlowReadException, ScriptException

View File

@ -1,4 +1,3 @@
import json
from textwrap import dedent
import re

View File

@ -2,7 +2,6 @@
from __future__ import (absolute_import, print_function, division)
import OpenSSL
import pytest
import traceback
import os
@ -468,13 +467,10 @@ class TestConnectionLost(_Http2TestBase, _Http2ServerBase):
])
done = False
ended_streams = 0
pushed_streams = 0
responses = 0
while not done:
try:
raw = b''.join(http2_read_raw_frame(client.rfile))
events = h2_conn.receive_data(raw)
h2_conn.receive_data(raw)
except:
break
client.wfile.write(h2_conn.data_to_send())

View File

@ -12,7 +12,7 @@ from unittest.case import SkipTest
import netlib.utils
import netlib.tutils
from mitmproxy import utils, controller
from mitmproxy import controller
from mitmproxy.models import (
ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow, TCPFlow
)

View File

@ -1,6 +1,5 @@
from __future__ import absolute_import, print_function, division
from io import BytesIO
import textwrap
from mock import Mock
from netlib.exceptions import HttpException, HttpSyntaxException, HttpReadDisconnect, TcpDisconnect
from netlib.http import Headers

View File

@ -1,4 +1,3 @@
import OpenSSL
import mock
import codecs

View File

@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division
from netlib.http import decoded, Headers
from netlib.tutils import tresp, raises
from netlib.http import decoded
from netlib.tutils import tresp
def _test_passthrough_attr(message, attr):

View File

@ -2,12 +2,10 @@ from __future__ import absolute_import, print_function, division
import email
import six
import time
from netlib.http import Headers
from netlib.http.cookies import CookieAttrs
from netlib.odict import ODict, ODictCaseless
from netlib.tutils import raises, tresp
from .test_message import _test_passthrough_attr, _test_decoded_attr

View File

@ -1,6 +1,5 @@
import ipaddress
from io import BytesIO
import socket
from netlib import socks, tcp, tutils

View File

@ -8,7 +8,6 @@ import threading
import mock
from OpenSSL import SSL
import OpenSSL
from netlib import tcp, certutils, tutils
from netlib.exceptions import InvalidCertificateException, TcpReadIncomplete, TlsException, \

View File

@ -1,6 +1,6 @@
from io import StringIO
import mock
from netlib import version_check, version
from netlib import version_check
@mock.patch("sys.exit")

View File

@ -2,7 +2,9 @@ import os
from netlib.http.http1 import read_response, read_request
from netlib import tcp, websockets, http, tutils
from netlib import tcp
from netlib import tutils
from netlib import websockets
from netlib.http import status_codes
from netlib.tutils import treq
from netlib.exceptions import *

View File

@ -5,7 +5,7 @@ from netlib import tcp
from netlib.http import user_agents
from pathod import language
from pathod.language import http2, base
from pathod.language import http2
import tutils

View File

@ -1,10 +1,10 @@
import StringIO
from pathod import log
from netlib.exceptions import TcpDisconnect
import netlib.tcp
import six
class DummyIO(StringIO.StringIO):
class DummyIO(six.StringIO):
def start_log(self, *args, **kwargs):
pass

View File

@ -1,12 +1,12 @@
import json
from six.moves import cStringIO as StringIO
import re
import OpenSSL
import pytest
from mock import Mock
from netlib import tcp, http, socks
from netlib.exceptions import HttpException, TcpException, NetlibException
from netlib import http
from netlib import tcp
from netlib.exceptions import NetlibException
from netlib.http import http1, http2
from pathod import pathoc, test, version, pathod, language

View File

@ -2,7 +2,7 @@ from six.moves import cStringIO as StringIO
import pytest
from pathod import pathod, version
from netlib import tcp, http
from netlib import tcp
from netlib.exceptions import HttpException, TlsException
import tutils

View File

@ -4,7 +4,10 @@ import shutil
from six.moves import cStringIO as StringIO
import netlib
from pathod import utils, test, pathoc, pathod, language
from pathod import language
from pathod import pathoc
from pathod import pathod
from pathod import test
from netlib import tcp
import requests