Merge branch 'master' into beautiful-json

.appveyor.yml

@@ -13,11 +13,13 @@ environment:
   SNAPSHOT_PASS:
     secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
 install:
-  - "pip install --user -U virtualenv"
+  - "pip install --user -U virtualenv codecov"
   - "dev.bat"
   - "python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\""
 test_script:
   - "py.test --timeout 60 --cov netlib --cov mitmproxy --cov pathod"
+after_test:
+  - "codecov"
 cache:
   - C:\Users\appveyor\AppData\Local\pip\cache
 deploy_script:

.travis.yml

@@ -14,20 +14,21 @@ matrix:
   fast_finish: true
   include:
     - python: 2.7
+      env: TOXENV=py27
     - python: 2.7
-      env: NO_ALPN=1
+      env: TOXENV=py27 NO_ALPN=1
+    - python: 3.5
+      env: TOXENV=py35
+    - python: 3.5
+      env: TOXENV=py35 NO_ALPN=1
     - language: generic
+      env: TOXENV=py27
       os: osx
       osx_image: xcode7.1
       git:
         depth: 9999999
-    - python: 3.5
-      env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py test/pathod/test_language_http.py"
-    - python: 3.5
-      env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py test/pathod/test_language_http.py" NO_ALPN=1
     - python: 2.7
-      env: DOCS=1
-      script: 'cd docs && SPHINXOPTS="-W" make -e html'
+      env: TOXENV=docs
   allow_failures:
     - python: pypy
 
@@ -39,23 +40,20 @@ install:
       brew outdated openssl || brew upgrade openssl
       brew install python
     fi
-  - pip install -U virtualenv
-  - ./dev.sh
-  - source venv/bin/activate
 
 before_script:
-  - "openssl version -a"
-  - "python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\""
-  - "flake8 --jobs 4 --count mitmproxy netlib pathod examples test"
+  - "pip install tox"
+  - "tox -e lint"
 
-script:
-  - "py.test --timeout 60 --cov netlib --cov mitmproxy --cov pathod test/$SCOPE"
+script: tox
 
 after_success:
-  - coveralls
   - |
     if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
     then
+        pip install -U virtualenv
+        ./dev.sh
+        source venv/bin/activate
         pip install -e ./release
         python ./release/rtool.py bdist
         python ./release/rtool.py upload-snapshot --bdist --wheel
@@ -71,7 +69,6 @@ notifications:
 
 cache:
   directories:
-    - $HOME/build/mitmproxy/mitmproxy/venv
+    - $HOME/build/mitmproxy/mitmproxy/.tox
     - $HOME/.cache/pip
     - $HOME/.pyenv
-    - $HOME/Library/Caches/pip

README.rst

@@ -1,7 +1,7 @@
 mitmproxy
 ^^^^^^^^^
 
-|travis| |coveralls| |latest_release| |python_versions|
+|travis| |coverage| |latest_release| |python_versions|
 
 This repository contains the **mitmproxy** and **pathod** projects, as well as
 their shared networking library, **netlib**.
@@ -141,8 +141,8 @@ good reason not to.
     :target: https://travis-ci.org/mitmproxy/mitmproxy
     :alt: Build Status
 
-.. |coveralls| image:: https://shields.mitmproxy.org/coveralls/mitmproxy/mitmproxy/master.svg
-    :target: https://coveralls.io/r/mitmproxy/mitmproxy
+.. |coverage| image:: https://codecov.io/gh/mitmproxy/mitmproxy/branch/master/graph/badge.svg
+    :target: https://codecov.io/gh/mitmproxy/mitmproxy
     :alt: Coverage Status
 
 .. |latest_release| image:: https://shields.mitmproxy.org/pypi/v/mitmproxy.svg

codecov.yml

@@ -0,0 +1 @@
+comment: off

netlib/strutils.py

@@ -120,13 +120,7 @@ def isMostlyBin(s):
 
 
 def isXML(s):
-    for i in s:
-        if i in "\n \t":
-            continue
-        elif i == "<":
-            return True
-        else:
-            return False
+    return s.strip().startswith("<")
 
 
 def clean_hanging_newline(t):

setup.py

@@ -97,9 +97,8 @@ setup(
             "ipaddress>=1.0.15, <1.1",
         ],
         'dev': [
-            "coveralls>=1.1, <1.2",
+            "tox>=2.3, <3",
             "mock>=2.0, <2.1",
-            "flake8>=2.5.4, <3",
             "pytest>=2.8.7, <2.10",
             "pytest-cov>=2.2.1, <2.3",
             "pytest-timeout>=1.0.0, <1.1",

test/netlib/test_strutils.py

@@ -1,6 +1,18 @@
 # coding=utf-8
+import six
 
-from netlib import strutils
+from netlib import strutils, tutils
+
+
+def test_native():
+    with tutils.raises(TypeError):
+        strutils.native(42)
+    if six.PY2:
+        assert strutils.native(u"foo") == b"foo"
+        assert strutils.native(b"foo") == b"foo"
+    else:
+        assert strutils.native(u"foo") == u"foo"
+        assert strutils.native(b"foo") == u"foo"
 
 
 def test_clean_bin():
@@ -29,6 +41,9 @@ def test_bytes_to_escaped_str():
     assert strutils.bytes_to_escaped_str(b"'") == r"\'"
     assert strutils.bytes_to_escaped_str(b'"') == r'"'
 
+    with tutils.raises(ValueError):
+        strutils.bytes_to_escaped_str(u"such unicode")
+
 
 def test_escaped_str_to_bytes():
     assert strutils.escaped_str_to_bytes("foo") == b"foo"
@@ -39,6 +54,13 @@ def test_escaped_str_to_bytes():
     assert strutils.escaped_str_to_bytes(u"&!?=\\\\)") == br"&!?=\)"
     assert strutils.escaped_str_to_bytes(u"ü") == b'\xc3\xbc'
 
+    if six.PY2:
+        with tutils.raises(ValueError):
+            strutils.escaped_str_to_bytes(42)
+    else:
+        with tutils.raises(ValueError):
+            strutils.escaped_str_to_bytes(b"very byte")
+
 
 def test_isBin():
     assert not strutils.isBin("testing\n\r")

test/pathod/test_language_writer.py

@@ -87,4 +87,4 @@ def test_write_values_after():
     s = BytesIO()
     r = next(language.parse_pathod("400:ia,'xx'"))
     language.serve(r, s, {})
-    assert s.getvalue().endswith('xx')
+    assert s.getvalue().endswith(b'xx')

tox.ini

@@ -1,15 +1,31 @@
 [tox]
-envlist = py27, py35, lint
+envlist = py27, py35, docs, lint
 
 [testenv]
-deps = -rrequirements.txt
+deps =
+  -rrequirements.txt
+  codecov>=2.0.5
+passenv = CI TRAVIS_BUILD_ID TRAVIS TRAVIS_BRANCH TRAVIS_JOB_NUMBER TRAVIS_PULL_REQUEST TRAVIS_JOB_ID TRAVIS_REPO_SLUG TRAVIS_COMMIT
 
 [testenv:py27]
-commands = py.test -n 8 --timeout 60 ./test
+commands =
+  py.test --cov netlib --cov mitmproxy --cov pathod --color=yes --timeout 60 ./test
+  codecov -e TOXENV
 
 [testenv:py35]
-commands = py.test -n 8 --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py
+# remove bash & pipe & grep hack after cryptography ships with openssl 1.1.0
+whitelist_externals = bash
+commands =
+  bash -c 'py.test --cov netlib --cov mitmproxy --cov pathod --color=yes --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py test/pathod/test_language_websocket.py 2>&1 | grep -v Cryptography_locking_cb'
+  codecov -e TOXENV
+
+[testenv:docs]
+basepython = python2.7
+whitelist_externals = make
+changedir = docs
+setenv = SPHINXOPTS="-W"
+commands = make -e html
 
 [testenv:lint]
-deps = flake8
-commands = flake8 --count mitmproxy netlib pathod examples test
+deps = flake8>=2.5.4, <3
+commands = flake8 --jobs 8 --count mitmproxy netlib pathod examples test