Merge branch 'master' into solidcore

.gitignore

@@ -16,3 +16,4 @@ build/
 node_modules
 bower_components
 *.map
+sslkeylogfile.log

.travis.yml

@@ -22,9 +22,9 @@ matrix:
       git:
         depth: 9999999
     - python: 3.5
-      env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py"
+      env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py ./test/pathod/test_log.py"
     - python: 3.5
-      env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py" NO_ALPN=1
+      env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py ./test/pathod/test_log.py" NO_ALPN=1
     - python: 2.7
       env: DOCS=1
       script: 'cd docs && make html'

mitmproxy/controller.py

@@ -12,6 +12,7 @@ class ControlError(Exception):
 
 
 class Master(object):
+
     """
     The master handles mitmproxy's main event loop.
     """
@@ -60,6 +61,7 @@ class Master(object):
 
 
 class ServerMaster(Master):
+
     """
     The ServerMaster adds server thread support to the master.
     """
@@ -86,6 +88,7 @@ class ServerMaster(Master):
 
 
 class ServerThread(threading.Thread):
+
     def __init__(self, server):
         self.server = server
         super(ServerThread, self).__init__()
@@ -97,6 +100,7 @@ class ServerThread(threading.Thread):
 
 
 class Channel(object):
+
     """
     The only way for the proxy server to communicate with the master
     is to use the channel it has been given.
@@ -137,6 +141,7 @@ class Channel(object):
 
 
 class DummyReply(object):
+
     """
     A reply object that does nothing. Useful when we need an object to seem
     like it has a channel, and during testing.
@@ -187,6 +192,7 @@ def handler(f):
 
 
 class Reply(object):
+
     """
     Messages sent through a channel are decorated with a "reply" attribute.
     This object is used to respond to the message through the return

mitmproxy/exceptions.py

@@ -13,6 +13,7 @@ import sys
 
 
 class ProxyException(Exception):
+
     """
     Base class for all exceptions thrown by mitmproxy.
     """
@@ -22,6 +23,7 @@ class ProxyException(Exception):
 
 
 class Kill(ProxyException):
+
     """
     Signal that both client and server connection(s) should be killed immediately.
     """
@@ -37,6 +39,7 @@ class TlsProtocolException(ProtocolException):
 
 
 class ClientHandshakeException(TlsProtocolException):
+
     def __init__(self, message, server):
         super(ClientHandshakeException, self).__init__(message)
         self.server = server
@@ -67,6 +70,7 @@ class ReplayException(ProxyException):
 
 
 class ScriptException(ProxyException):
+
     @classmethod
     def from_exception_context(cls, cut_tb=1):
         """

mitmproxy/filt.py

@@ -38,6 +38,7 @@ import pyparsing as pp
 
 
 class _Token(object):
+
     def dump(self, indent=0, fp=sys.stdout):
         print("{spacing}{name}{expr}".format(
             spacing="\t" * indent,

mitmproxy/flow.py

@@ -8,13 +8,13 @@ import hashlib
 import sys
 
 import six
-from six.moves import http_cookies, http_cookiejar, urllib
+from six.moves import http_cookiejar
+from six.moves import urllib
 import os
 import re
 
-from typing import List, Optional, Set
 
-from netlib import wsgi, odict
+from netlib import wsgi
 from netlib.exceptions import HttpException
 from netlib.http import Headers, http1, cookies
 from netlib.utils import clean_bin

mitmproxy/flow_export.py

@@ -5,7 +5,9 @@ import netlib.http
 from netlib.utils import parse_content_type
 
 import re
-from six.moves.urllib.parse import urlparse, quote, quote_plus
+from six.moves.urllib.parse import quote
+from six.moves.urllib.parse import quote_plus
+
 
 def curl_command(flow):
     data = "curl "
@@ -123,7 +125,6 @@ def locust_code(flow):
             max_wait = 3000
 """).strip()
 
-
     components = map(lambda x: quote(x, safe=""), flow.request.path_components)
     file_name = "_".join(components)
     name = re.sub('\W|^(?=\d)', '_', file_name)

mitmproxy/models/connections.py

@@ -10,6 +10,7 @@ from .. import stateobject, utils
 
 
 class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
+
     """
     A client connection
 
@@ -21,6 +22,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         timestamp_ssl_setup: TLS established timestamp
         timestamp_end: Connection end timestamp
     """
+
     def __init__(self, client_connection, address, server):
         # Eventually, this object is restored from state. We don't have a
         # connection then.
@@ -101,6 +103,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
 
 
 class ServerConnection(tcp.TCPClient, stateobject.StateObject):
+
     """
     A server connection
 
@@ -117,6 +120,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
         timestamp_ssl_setup: TLS established timestamp
         timestamp_end: Connection end timestamp
     """
+
     def __init__(self, address, source_address=None):
         tcp.TCPClient.__init__(self, address, source_address)
 

mitmproxy/models/http.py

@@ -109,6 +109,7 @@ class HTTPRequest(MessageMixin, Request):
 
 
 class HTTPResponse(MessageMixin, Response):
+
     """
     A mitmproxy HTTP response.
     This is a very thin wrapper on top of :py:class:`netlib.http.Response` and

mitmproxy/models/tcp.py

@@ -6,6 +6,7 @@ from .flow import Flow
 
 
 class TCPMessage(Serializable):
+
     def __init__(self, from_client, content, timestamp=None):
         self.content = content
         self.from_client = from_client
@@ -33,6 +34,7 @@ class TCPMessage(Serializable):
 
 
 class TCPFlow(Flow):
+
     """
     A TCPFlow is a simplified representation of a TCP session.
     """

mitmproxy/protocol/rawtcp.py

@@ -1,14 +1,10 @@
 from __future__ import (absolute_import, print_function, division)
 import socket
-import six
-import sys
 
 from OpenSSL import SSL
 from netlib.exceptions import TcpException
 
 from netlib.tcp import ssl_read_select
-from netlib.utils import clean_bin
-from ..exceptions import ProtocolException
 from ..models import Error
 from ..models.tcp import TCPFlow, TCPMessage
 

mitmproxy/protocol/tls.py

@@ -312,6 +312,7 @@ class TlsClientHello(object):
 
 
 class TlsLayer(Layer):
+
     """
     The TLS layer implements transparent TLS connections.
 

mitmproxy/script/script.py

@@ -6,7 +6,6 @@ by the mitmproxy-specific ScriptContext.
 # Do not import __future__ here, this would apply transitively to the inline scripts.
 import os
 import shlex
-import traceback
 import sys
 
 import six

mitmproxy/tnetstring.py

@@ -68,6 +68,7 @@ like so::
 
 """
 import six
+from collections import deque
 
 __ver_major__ = 0
 __ver_minor__ = 2
@@ -77,9 +78,6 @@ __version__ = "%d.%d.%d%s" % (
     __ver_major__, __ver_minor__, __ver_patch__, __ver_sub__)
 
 
-from collections import deque
-
-
 def dumps(value, encoding=None):
     """dumps(object,encoding=None) -> string
 

netlib/http/cookies.py

@@ -3,7 +3,6 @@ import re
 from email.utils import parsedate_tz, formatdate, mktime_tz
 
 from netlib.multidict import ImmutableMultiDict
-from .. import odict
 
 """
 A flexible module for cookie parsing and manipulation.
@@ -28,6 +27,7 @@ variants. Serialization follows RFC6265.
 
 # TODO: Disallow LHS-only Cookie values
 
+
 def _read_until(s, start, term):
     """
         Read until one of the characters in term is reached.

netlib/http/headers.py

@@ -14,12 +14,18 @@ from ..utils import always_bytes
 # See also: http://lucumr.pocoo.org/2013/7/2/the-updated-guide-to-unicode/
 
 if six.PY2:  # pragma: no cover
-    _native = lambda x: x
+    def _native(x):
-    _always_bytes = lambda x: x
+        return x
+
+    def _always_bytes(x):
+        return x
 else:
     # While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
-    _native = lambda x: x.decode("utf-8", "surrogateescape")
+    def _native(x):
-    _always_bytes = lambda x: always_bytes(x, "utf-8", "surrogateescape")
+        return x.decode("utf-8", "surrogateescape")
+
+    def _always_bytes(x):
+        return always_bytes(x, "utf-8", "surrogateescape")
 
 
 class Headers(MultiDict):

netlib/http/http1/assemble.py

@@ -1,9 +1,9 @@
 from __future__ import absolute_import, print_function, division
 
 from ... import utils
-import itertools
 from ...exceptions import HttpException
 
+
 def assemble_request(request):
     if request.content is None:
         raise HttpException("Cannot assemble flow with missing content")

netlib/http/message.py

@@ -4,17 +4,23 @@ import warnings
 
 import six
 
-from ..multidict import MultiDict
 from .headers import Headers
 from .. import encoding, utils
+from ..utils import always_bytes
 
 if six.PY2:  # pragma: no cover
-    _native = lambda x: x
+    def _native(x):
-    _always_bytes = lambda x: x
+        return x
+
+    def _always_bytes(x):
+        return x
 else:
-    # While the HTTP head _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
+    # While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
-    _native = lambda x: x.decode("utf-8", "surrogateescape")
+    def _native(x):
-    _always_bytes = lambda x: utils.always_bytes(x, "utf-8", "surrogateescape")
+        return x.decode("utf-8", "surrogateescape")
+
+    def _always_bytes(x):
+        return always_bytes(x, "utf-8", "surrogateescape")
 
 
 class MessageData(utils.Serializable):

netlib/http/request.py

@@ -1,14 +1,12 @@
 from __future__ import absolute_import, print_function, division
 
 import re
-import warnings
 
 import six
 from six.moves import urllib
 
 from netlib import utils
 from netlib.http import cookies
-from netlib.odict import ODict
 from .. import encoding
 from ..multidict import MultiDictView
 from .headers import Headers

netlib/multidict.py

@@ -2,7 +2,6 @@ from __future__ import absolute_import, print_function, division
 
 from abc import ABCMeta, abstractmethod
 
-from typing import Tuple, TypeVar
 
 try:
     from collections.abc import MutableMapping

netlib/tcp.py

@@ -71,6 +71,7 @@ sslversion_choices = {
     "TLSv1_2": (SSL.TLSv1_2_METHOD, SSL_BASIC_OPTIONS),
 }
 
+
 class SSLKeyLogger(object):
 
     def __init__(self, filename):

netlib/tutils.py

@@ -91,7 +91,6 @@ class RaisesContext(object):
 
 test_data = utils.Data(__name__)
 # FIXME: Temporary workaround during repo merge.
-import os
 test_data.dirname = os.path.join(test_data.dirname, "..", "test", "netlib")
 
 

netlib/version_check.py

@@ -10,7 +10,6 @@ import os.path
 import six
 
 import OpenSSL
-from . import version
 
 PYOPENSSL_MIN_VERSION = (0, 15)
 

netlib/websockets/protocol.py

@@ -1,24 +1,23 @@
+"""
+Colleciton of utility functions that implement small portions of the RFC6455
+WebSockets Protocol Useful for building WebSocket clients and servers.
 
+Emphassis is on readabilty, simplicity and modularity, not performance or
+completeness
 
+This is a work in progress and does not yet contain all the utilites need to
+create fully complient client/servers #
+Spec: https://tools.ietf.org/html/rfc6455
 
-# Colleciton of utility functions that implement small portions of the RFC6455
+The magic sha that websocket servers must know to prove they understand
-# WebSockets Protocol Useful for building WebSocket clients and servers.
+RFC6455
-#
+"""
-# Emphassis is on readabilty, simplicity and modularity, not performance or
-# completeness
-#
-# This is a work in progress and does not yet contain all the utilites need to
-# create fully complient client/servers #
-# Spec: https://tools.ietf.org/html/rfc6455
 
-# The magic sha that websocket servers must know to prove they understand
-# RFC6455
 from __future__ import absolute_import
 import base64
 import hashlib
 import os
 
-import binascii
 import six
 from ..http import Headers
 
@@ -95,21 +94,18 @@ class WebsocketsProtocol(object):
             upgrade="websocket"
         )
 
-
     @classmethod
     def check_client_handshake(self, headers):
         if headers.get("upgrade") != "websocket":
             return
         return headers.get("sec-websocket-key")
 
-
     @classmethod
     def check_server_handshake(self, headers):
         if headers.get("upgrade") != "websocket":
             return
         return headers.get("sec-websocket-accept")
 
-
     @classmethod
     def create_server_nonce(self, client_nonce):
         return base64.b64encode(hashlib.sha1(client_nonce + websockets_magic).digest())

netlib/wsgi.py

@@ -1,15 +1,15 @@
 from __future__ import (absolute_import, print_function, division)
-from io import BytesIO, StringIO
+
-import urllib
 import time
 import traceback
-
 import six
+from io import BytesIO
 from six.moves import urllib
 
 from netlib.utils import always_bytes, native
 from . import http, tcp
 
+
 class ClientConn(object):
 
     def __init__(self, address):
@@ -154,7 +154,7 @@ class WSGIAdaptor(object):
                 write(i)
             if not state["headers_sent"]:
                 write(b"")
-        except Exception as e:
+        except Exception:
             try:
                 s = traceback.format_exc()
                 errs.write(s.encode("utf-8", "replace"))

pathod/language/http.py

@@ -11,6 +11,7 @@ from . import base, exceptions, actions, message
 # instead of duplicating the HTTP on-the-wire representation here.
 # see http2 language for an example
 
+
 class WS(base.CaselessLiteral):
     TOK = "ws"
 

pathod/language/http2.py

@@ -27,6 +27,7 @@ from . import base, message
         h2f:42:DATA:END_STREAM,PADDED:0x1234567:'content body payload'
 """
 
+
 def get_header(val, headers):
     """
         Header keys may be Values, so we have to "generate" them as we try the
@@ -48,6 +49,7 @@ class _HeaderMixin(object):
             self.value.get_generator(settings),
         )
 
+
 class _HTTP2Message(message.Message):
     @property
     def actions(self):
@@ -287,13 +289,10 @@ class Request(_HTTP2Message):
     def spec(self):
         return ":".join([i.spec() for i in self.tokens])
 
+
 def make_error_response(reason, body=None):
     tokens = [
         StatusCode("800"),
         Body(base.TokValueLiteral("pathod error: " + (body or reason))),
     ]
     return Response(tokens)
-
-
-# class Frame(message.Message):
-#     pass

pathod/language/websockets.py

@@ -1,4 +1,3 @@
-import os
 import random
 import string
 import netlib.websockets

pathod/language/writer.py

@@ -1,6 +1,5 @@
 import time
 from netlib.exceptions import TcpDisconnect
-import netlib.tcp
 
 BLOCKSIZE = 1024
 # It's not clear what the upper limit for time.sleep is. It's lower than the

pathod/pathoc.py

@@ -72,7 +72,6 @@ class SSLInfo(object):
         return "\n".join(parts)
 
 
-
 class WebsocketFrameReader(threading.Thread):
 
     def __init__(

pathod/pathod.py

@@ -112,7 +112,6 @@ class PathodHandler(tcp.BaseHandler):
             return None, response_log
         return self.handle_http_request, response_log
 
-
     def handle_http_request(self, logger):
         """
             Returns a (handler, log) tuple.

pathod/pathod_cmdline.py

@@ -177,7 +177,6 @@ def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
         help="Output all received & sent HTTP/2 frames"
     )
 
-
     args = parser.parse_args(argv[1:])
 
     args.ssl_version, args.ssl_options = tcp.sslversion_choices[args.ssl_version]

pathod/protocols/http.py

@@ -1,6 +1,6 @@
-from netlib import tcp, wsgi
+from netlib import wsgi
-from netlib.exceptions import HttpReadDisconnect, TlsException
+from netlib.exceptions import TlsException
-from netlib.http import http1, Request
+from netlib.http import http1
 from .. import version, language
 
 

pathod/protocols/http2.py

@@ -1,5 +1,6 @@
 from netlib.http import http2
-from .. import version, app, language, utils, log
+from .. import language
+
 
 class HTTP2Protocol:
 

test/mitmproxy/test_examples.py

@@ -3,7 +3,7 @@ import json
 import os
 from contextlib import contextmanager
 
-from mitmproxy import utils, script
+from mitmproxy import script
 from mitmproxy.proxy import config
 import netlib.utils
 from netlib import tutils as netutils

test/mitmproxy/test_flow.py

@@ -4,7 +4,6 @@ from six.moves import cStringIO as StringIO
 import mock
 
 import netlib.utils
-from netlib import odict
 from netlib.http import Headers
 from mitmproxy import filt, controller, tnetstring, flow
 from mitmproxy.exceptions import FlowReadException, ScriptException

test/mitmproxy/test_flow_export.py

@@ -1,4 +1,3 @@
-import json
 from textwrap import dedent
 import re
 

test/mitmproxy/test_protocol_http2.py

@@ -2,7 +2,6 @@
 
 from __future__ import (absolute_import, print_function, division)
 
-import OpenSSL
 import pytest
 import traceback
 import os
@@ -468,13 +467,10 @@ class TestConnectionLost(_Http2TestBase, _Http2ServerBase):
         ])
 
         done = False
-        ended_streams = 0
-        pushed_streams = 0
-        responses = 0
         while not done:
             try:
                 raw = b''.join(http2_read_raw_frame(client.rfile))
-                events = h2_conn.receive_data(raw)
+                h2_conn.receive_data(raw)
             except:
                 break
             client.wfile.write(h2_conn.data_to_send())

test/mitmproxy/tutils.py

@@ -12,7 +12,7 @@ from unittest.case import SkipTest
 
 import netlib.utils
 import netlib.tutils
-from mitmproxy import utils, controller
+from mitmproxy import controller
 from mitmproxy.models import (
     ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow, TCPFlow
 )

test/netlib/http/http1/test_read.py

@@ -1,6 +1,5 @@
 from __future__ import absolute_import, print_function, division
 from io import BytesIO
-import textwrap
 from mock import Mock
 from netlib.exceptions import HttpException, HttpSyntaxException, HttpReadDisconnect, TcpDisconnect
 from netlib.http import Headers

test/netlib/http/http2/test_connections.py

@@ -1,4 +1,3 @@
-import OpenSSL
 import mock
 import codecs
 

test/netlib/http/test_message.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, print_function, division
 
-from netlib.http import decoded, Headers
+from netlib.http import decoded
-from netlib.tutils import tresp, raises
+from netlib.tutils import tresp
 
 
 def _test_passthrough_attr(message, attr):

test/netlib/http/test_response.py

@@ -2,12 +2,10 @@ from __future__ import absolute_import, print_function, division
 
 import email
 
-import six
 import time
 
 from netlib.http import Headers
 from netlib.http.cookies import CookieAttrs
-from netlib.odict import ODict, ODictCaseless
 from netlib.tutils import raises, tresp
 from .test_message import _test_passthrough_attr, _test_decoded_attr
 

test/netlib/test_socks.py

@@ -1,6 +1,5 @@
 import ipaddress
 from io import BytesIO
-import socket
 from netlib import socks, tcp, tutils
 
 

test/netlib/test_tcp.py

@@ -8,7 +8,6 @@ import threading
 import mock
 
 from OpenSSL import SSL
-import OpenSSL
 
 from netlib import tcp, certutils, tutils
 from netlib.exceptions import InvalidCertificateException, TcpReadIncomplete, TlsException, \

test/netlib/test_version_check.py

@@ -1,6 +1,6 @@
 from io import StringIO
 import mock
-from netlib import version_check, version
+from netlib import version_check
 
 
 @mock.patch("sys.exit")

test/netlib/websockets/test_websockets.py

@@ -2,7 +2,9 @@ import os
 
 from netlib.http.http1 import read_response, read_request
 
-from netlib import tcp, websockets, http, tutils
+from netlib import tcp
+from netlib import tutils
+from netlib import websockets
 from netlib.http import status_codes
 from netlib.tutils import treq
 from netlib.exceptions import *

test/pathod/test_language_http2.py

@@ -5,7 +5,7 @@ from netlib import tcp
 from netlib.http import user_agents
 
 from pathod import language
-from pathod.language import http2, base
+from pathod.language import http2
 import tutils
 
 

test/pathod/test_log.py

@@ -1,10 +1,10 @@
-import StringIO
 from pathod import log
 from netlib.exceptions import TcpDisconnect
-import netlib.tcp
+
+import six
 
 
-class DummyIO(StringIO.StringIO):
+class DummyIO(six.StringIO):
 
     def start_log(self, *args, **kwargs):
         pass

test/pathod/test_pathoc.py

@@ -1,12 +1,12 @@
 import json
 from six.moves import cStringIO as StringIO
 import re
-import OpenSSL
 import pytest
 from mock import Mock
 
-from netlib import tcp, http, socks
+from netlib import http
-from netlib.exceptions import HttpException, TcpException, NetlibException
+from netlib import tcp
+from netlib.exceptions import NetlibException
 from netlib.http import http1, http2
 
 from pathod import pathoc, test, version, pathod, language

test/pathod/test_pathod.py

@@ -2,7 +2,7 @@ from six.moves import cStringIO as StringIO
 import pytest
 
 from pathod import pathod, version
-from netlib import tcp, http
+from netlib import tcp
 from netlib.exceptions import HttpException, TlsException
 import tutils
 

test/pathod/tutils.py

@@ -4,7 +4,10 @@ import shutil
 from six.moves import cStringIO as StringIO
 
 import netlib
-from pathod import utils, test, pathoc, pathod, language
+from pathod import language
+from pathod import pathoc
+from pathod import pathod
+from pathod import test
 from netlib import tcp
 import requests