more style cleanup

Use this to check:
flake8 --count mitmproxy netlib pathod examples test

.travis.yml

@@ -46,7 +46,7 @@ install:
 before_script:
   - "openssl version -a"
   - "python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\""
-  - "[[ $(flake8 --exclude=mitmproxy/contrib,mitmproxy/console --ignore E251,C901 --count --exit-zero mitmproxy netlib pathod examples test | wc -l) -le 159 ]]"
+  - "[[ $(flake8 --count --exit-zero mitmproxy netlib pathod examples test | wc -l) -le 72 ]]"
 
 script:
   - "py.test --timeout 60 --cov netlib --cov mitmproxy --cov pathod ./test/$SCOPE"

mitmproxy/controller.py

@@ -68,13 +68,13 @@ class Master(object):
             while True:
                 mtype, obj = self.event_queue.get(timeout=timeout)
                 if mtype not in Events:
-                    raise exceptions.ControlException("Unknown event %s"%repr(mtype))
+                    raise exceptions.ControlException("Unknown event %s" % repr(mtype))
                 handle_func = getattr(self, mtype)
                 if not hasattr(handle_func, "func_dict"):
-                    raise exceptions.ControlException("Handler %s not a function"%mtype)
+                    raise exceptions.ControlException("Handler %s not a function" % mtype)
                 if not handle_func.func_dict.get("__handler"):
                     raise exceptions.ControlException(
-                        "Handler function %s is not decorated with controller.handler"%(
+                        "Handler function %s is not decorated with controller.handler" % (
                             handle_func
                         )
                     )

mitmproxy/flow_export.py

@@ -135,7 +135,8 @@ def locust_code(flow):
     args = ""
     headers = ""
     if flow.request.headers:
-        lines = ["            '%s': '%s',\n" % (k, v) for k, v in flow.request.headers.fields if k.lower() not in ["host", "cookie"]]
+        lines = [(k, v) for k, v in flow.request.headers.fields if k.lower() not in ["host", "cookie"]]
+        lines = ["            '%s': '%s',\n" % (k, v) for k, v in lines]
         headers += "\n        headers = {\n%s        }\n" % "".join(lines)
         args += "\n            headers=headers,"
 

netlib/http/http2/connections.py

@@ -356,7 +356,7 @@ class HTTP2Protocol(object):
         frms = [frm_cls(
             flags=[],
             stream_id=stream_id,
-            data=header_block_fragment[i:i+chunk_size]) for frm_cls, i in frame_cls(chunks)]
+            data=header_block_fragment[i:i + chunk_size]) for frm_cls, i in frame_cls(chunks)]
 
         frms[-1].flags.add('END_HEADERS')
         if end_stream:
@@ -377,7 +377,7 @@ class HTTP2Protocol(object):
         frms = [frame.DataFrame(
             flags=[],
             stream_id=stream_id,
-            data=body[i:i+chunk_size]) for i in chunks]
+            data=body[i:i + chunk_size]) for i in chunks]
         frms[-1].flags.add('END_STREAM')
 
         if self.dump_frames:  # pragma no cover

setup.cfg

@@ -1,11 +1,8 @@
 [flake8]
-max-line-length = 120
+max-line-length = 130
-max-complexity = 20
+max-complexity = 25
-
+ignore = E251,C901
-[pep8]
+exclude = mitmproxy/contrib/*
-max-line-length = 120
-exclude = */contrib/*
-ignore = E251
 
 [pytest]
 testpaths = test

test/mitmproxy/test_flow.py

@@ -54,7 +54,7 @@ class TestStickyCookieState:
         assert s.domain_match("google.com", ".google.com")
 
     def test_response(self):
-        c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; "\
+        c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " \
             "Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; "
 
         s, f = self._response(c, "host")
@@ -388,22 +388,22 @@ class TestFlow(object):
         del b["id"]
         assert a == b
         assert not f == f2
-        assert not f is f2
+        assert f is not f2
         assert f.request.get_state() == f2.request.get_state()
-        assert not f.request is f2.request
+        assert f.request is not f2.request
         assert f.request.headers == f2.request.headers
-        assert not f.request.headers is f2.request.headers
+        assert f.request.headers is not f2.request.headers
         assert f.response.get_state() == f2.response.get_state()
-        assert not f.response is f2.response
+        assert f.response is not f2.response
 
         f = tutils.tflow(err=True)
         f2 = f.copy()
-        assert not f is f2
+        assert f is not f2
-        assert not f.request is f2.request
+        assert f.request is not f2.request
         assert f.request.headers == f2.request.headers
-        assert not f.request.headers is f2.request.headers
+        assert f.request.headers is not f2.request.headers
         assert f.error.get_state() == f2.error.get_state()
-        assert not f.error is f2.error
+        assert f.error is not f2.error
 
     def test_match(self):
         f = tutils.tflow(resp=True)
@@ -979,7 +979,7 @@ class TestFlowMaster:
         fm.request(f)
         fm.response(f)
         assert fm.stickycookie_state.jar
-        assert not "cookie" in f.request.headers
+        assert "cookie" not in f.request.headers
         f = f.copy()
         fm.request(f)
         assert f.request.headers["cookie"] == "foo=bar"
@@ -1000,7 +1000,7 @@ class TestFlowMaster:
 
         f = tutils.tflow(resp=True)
         assert fm.stickyauth_state.hosts
-        assert not "authorization" in f.request.headers
+        assert "authorization" not in f.request.headers
         fm.request(f)
         assert f.request.headers["authorization"] == "foo"
 
@@ -1070,8 +1070,8 @@ class TestRequest:
         r.headers["if-modified-since"] = "test"
         r.headers["if-none-match"] = "test"
         r.anticache()
-        assert not "if-modified-since" in r.headers
+        assert "if-modified-since" not in r.headers
-        assert not "if-none-match" in r.headers
+        assert "if-none-match" not in r.headers
 
     def test_replace(self):
         r = HTTPRequest.wrap(netlib.tutils.treq())
@@ -1080,7 +1080,7 @@ class TestRequest:
         r.content = "afoob"
         assert r.replace("foo(?i)", "boo") == 4
         assert r.path == "path/boo"
-        assert not "foo" in r.content
+        assert "foo" not in r.content
         assert r.headers["boo"] == "boo"
 
     def test_constrain_encoding(self):
@@ -1122,7 +1122,7 @@ class TestResponse:
         r.headers["Foo"] = "fOo"
         r.content = "afoob"
         assert r.replace("foo(?i)", "boo") == 3
-        assert not "foo" in r.content
+        assert "foo" not in r.content
         assert r.headers["boo"] == "boo"
 
     def test_get_content_type(self):
@@ -1154,11 +1154,9 @@ class TestError:
 
 
 class TestClientConnection:
-
     def test_state(self):
-
         c = tutils.tclient_conn()
-        assert ClientConnection.from_state(c.get_state()).get_state() ==\
+        assert ClientConnection.from_state(c.get_state()).get_state() == \
             c.get_state()
 
         c2 = tutils.tclient_conn()

test/mitmproxy/test_protocol_http2.py

@@ -6,10 +6,17 @@ import pytest
 import traceback
 import os
 import tempfile
+import h2
 
 from mitmproxy.proxy.config import ProxyConfig
 from mitmproxy.cmdline import APP_HOST, APP_PORT
 
+import netlib
+from ..netlib import tservers as netlib_tservers
+from netlib.utils import http2_read_raw_frame
+
+from . import tservers
+
 import logging
 logging.getLogger("hyper.packages.hpack.hpack").setLevel(logging.WARNING)
 logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(logging.WARNING)
@@ -18,13 +25,6 @@ logging.getLogger("passlib.registry").setLevel(logging.WARNING)
 logging.getLogger("PIL.Image").setLevel(logging.WARNING)
 logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING)
 
-import netlib
-from ..netlib import tservers as netlib_tservers
-from netlib.utils import http2_read_raw_frame
-
-import h2
-
-from . import tservers
 
 requires_alpn = pytest.mark.skipif(
     not netlib.tcp.HAS_ALPN,

test/mitmproxy/tools/memoryleak.py

@@ -3,8 +3,8 @@ import threading
 from pympler import muppy, refbrowser
 from OpenSSL import SSL
 # import os
-#os.environ["TK_LIBRARY"] = r"C:\Python27\tcl\tcl8.5"
+# os.environ["TK_LIBRARY"] = r"C:\Python27\tcl\tcl8.5"
-#os.environ["TCL_LIBRARY"] = r"C:\Python27\tcl\tcl8.5"
+# os.environ["TCL_LIBRARY"] = r"C:\Python27\tcl\tcl8.5"
 
 # Also noteworthy: guppy, objgraph
 

test/netlib/http/http1/test_assemble.py

@@ -10,11 +10,11 @@ from netlib.tutils import treq, raises, tresp
 
 
 def test_assemble_request():
-    c = assemble_request(treq()) == (
+    assert assemble_request(treq()) == (
         b"GET /path HTTP/1.1\r\n"
         b"header: qvalue\r\n"
-        b"Host: address:22\r\n"
+        b"content-length: 7\r\n"
-        b"Content-Length: 7\r\n"
+        b"host: address:22\r\n"
         b"\r\n"
         b"content"
     )
@@ -32,10 +32,10 @@ def test_assemble_request_head():
 
 
 def test_assemble_response():
-    c = assemble_response(tresp()) == (
+    assert assemble_response(tresp()) == (
         b"HTTP/1.1 200 OK\r\n"
+        b"content-length: 7\r\n"
         b"header-response: svalue\r\n"
-        b"Content-Length: 7\r\n"
         b"\r\n"
         b"message"
     )

test/netlib/http/http1/test_read.py

@@ -105,6 +105,7 @@ class TestReadBody(object):
         rfile = BytesIO(b"123456")
         assert list(read_body(rfile, -1, max_chunk_size=1)) == [b"1", b"2", b"3", b"4", b"5", b"6"]
 
+
 def test_connection_close():
     headers = Headers()
     assert connection_close(b"HTTP/1.0", headers)
@@ -120,6 +121,7 @@ def test_connection_close():
     assert connection_close(b"HTTP/1.0", headers)
     assert not connection_close(b"HTTP/1.1", headers)
 
+
 def test_expected_http_body_size():
     # Expect: 100-continue
     assert expected_http_body_size(
@@ -202,6 +204,7 @@ def test_read_request_line():
     with raises(HttpReadDisconnect):
         t(b"")
 
+
 def test_parse_authority_form():
     assert _parse_authority_form(b"foo:42") == (b"foo", 42)
     with raises(HttpSyntaxException):
@@ -301,6 +304,7 @@ class TestReadHeaders(object):
         headers = self._read(data)
         assert headers.fields == ((b"bar", b""),)
 
+
 def test_read_chunked():
     req = treq(content=None)
     req.headers["Transfer-Encoding"] = "chunked"

test/netlib/http/http2/test_connections.py

@@ -1,7 +1,7 @@
 import mock
 import codecs
 
-from hyperframe.frame import *
+from hyperframe import frame
 
 from netlib import tcp, http, utils
 from netlib.tutils import raises
@@ -10,6 +10,7 @@ from netlib.http.http2.connections import HTTP2Protocol, TCPHandler
 
 from ... import tservers
 
+
 class TestTCPHandlerWrapper:
     def test_wrapped(self):
         h = TCPHandler(rfile='foo', wfile='bar')
@@ -213,19 +214,19 @@ class TestApplySettings(tservers.ServerTestBase):
         protocol = HTTP2Protocol(c)
 
         protocol._apply_settings({
-            SettingsFrame.ENABLE_PUSH: 'foo',
+            frame.SettingsFrame.ENABLE_PUSH: 'foo',
-            SettingsFrame.MAX_CONCURRENT_STREAMS: 'bar',
+            frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 'bar',
-            SettingsFrame.INITIAL_WINDOW_SIZE: 'deadbeef',
+            frame.SettingsFrame.INITIAL_WINDOW_SIZE: 'deadbeef',
         })
 
         assert c.rfile.safe_read(2) == b"OK"
 
         assert protocol.http2_settings[
-            SettingsFrame.ENABLE_PUSH] == 'foo'
+            frame.SettingsFrame.ENABLE_PUSH] == 'foo'
         assert protocol.http2_settings[
-            SettingsFrame.MAX_CONCURRENT_STREAMS] == 'bar'
+            frame.SettingsFrame.MAX_CONCURRENT_STREAMS] == 'bar'
         assert protocol.http2_settings[
-            SettingsFrame.INITIAL_WINDOW_SIZE] == 'deadbeef'
+            frame.SettingsFrame.INITIAL_WINDOW_SIZE] == 'deadbeef'
 
 
 class TestCreateHeaders(object):
@@ -257,7 +258,7 @@ class TestCreateHeaders(object):
             (b'server', b'version')])
 
         protocol = HTTP2Protocol(self.c)
-        protocol.http2_settings[SettingsFrame.MAX_FRAME_SIZE] = 8
+        protocol.http2_settings[frame.SettingsFrame.MAX_FRAME_SIZE] = 8
         bytes = protocol._create_headers(headers, 1, end_stream=True)
         assert len(bytes) == 3
         assert bytes[0] == codecs.decode('000008010100000001828487408294e783', 'hex_codec')
@@ -280,7 +281,7 @@ class TestCreateBody(object):
 
     def test_create_body_multiple_frames(self):
         protocol = HTTP2Protocol(self.c)
-        protocol.http2_settings[SettingsFrame.MAX_FRAME_SIZE] = 5
+        protocol.http2_settings[frame.SettingsFrame.MAX_FRAME_SIZE] = 5
         bytes = protocol._create_body(b'foobarmehm42', 1)
         assert len(bytes) == 3
         assert bytes[0] == codecs.decode('000005000000000001666f6f6261', 'hex_codec')

test/netlib/websockets/test_websockets.py

@@ -7,10 +7,11 @@ from netlib import tutils
 from netlib import websockets
 from netlib.http import status_codes
 from netlib.tutils import treq
-from netlib.exceptions import *
+from netlib import exceptions
 
 from .. import tservers
 
+
 class WebSocketsEchoHandler(tcp.BaseHandler):
 
     def __init__(self, connection, address, server):
@@ -176,7 +177,7 @@ class TestBadHandshake(tservers.ServerTestBase):
     handler = BadHandshakeHandler
 
     def test(self):
-        with tutils.raises(TcpDisconnect):
+        with tutils.raises(exceptions.TcpDisconnect):
             client = WebSocketsClient(("127.0.0.1", self.port))
             client.connect()
             client.send_message(b"hello")

test/pathod/test_language_http2.py

@@ -141,7 +141,6 @@ class TestRequest:
         assert isinstance(r.tokens[2], http2.NestedResponse)
         assert r.values(default_settings())
 
-
     def test_render_with_body(self):
         s = StringIO()
         r = parse_request("GET:'/foo':bfoobar")

test/pathod/test_pathoc.py

@@ -147,7 +147,7 @@ class TestDaemon(_TestDaemon):
         tutils.raises("ssl handshake", c.connect)
 
     def test_showssl(self):
-        assert not "certificate chain" in self.tval(
+        assert "certificate chain" not in self.tval(
             ["get:/p/200"],
             showssl=True)
 
@@ -170,7 +170,7 @@ class TestDaemon(_TestDaemon):
             showresp=True,
             timeout=1
         )
-        assert not "HTTP" in self.tval(
+        assert "HTTP" not in self.tval(
             ["get:'/p/200:p3,100'"],
             showresp=True,
             timeout=1,

test/pathod/test_pathoc_cmdline.py

@@ -29,13 +29,13 @@ def test_pathoc(perror):
     assert a.connect_to == ["foo", 10]
 
     a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2"])
-    assert a.use_http2 == True
+    assert a.use_http2 is True
-    assert a.ssl == True
+    assert a.ssl is True
 
     a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2-skip-connection-preface"])
-    assert a.use_http2 == True
+    assert a.use_http2 is True
-    assert a.ssl == True
+    assert a.ssl is True
-    assert a.http2_skip_connection_preface == True
+    assert a.http2_skip_connection_preface is True
 
     a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"])
     assert perror.called

test/pathod/tutils.py

@@ -1,15 +1,18 @@
 import tempfile
 import re
 import shutil
+import requests
 from six.moves import cStringIO as StringIO
 
-import netlib
+from netlib import tcp
+from netlib import utils
+from netlib import tutils
+
 from pathod import language
 from pathod import pathoc
 from pathod import pathod
 from pathod import test
-from netlib import tcp
+
-import requests
 
 def treader(bytes):
     """
@@ -115,11 +118,11 @@ class DaemonTests(object):
         return ret, logfp.getvalue()
 
 
-tmpdir = netlib.tutils.tmpdir
+tmpdir = tutils.tmpdir
 
-raises = netlib.tutils.raises
+raises = tutils.raises
 
-test_data = netlib.utils.Data(__name__)
+test_data = utils.Data(__name__)
 
 
 def render(r, settings=language.Settings()):