Merge pull request #1301 from mitmproxy/simplify-tox

Simplify tox

.appveyor.yml

@@ -2,11 +2,13 @@ version: '{build}'
 build: off  # Not a C# project
 
 environment:
+  CI_DEPS: codecov>=2.0.5
+  CI_COMMANDS: codecov
   matrix:
     - PYTHON: "C:\\Python27"
-      TOXENV: "py27-ci"
+      TOXENV: "py27"
     - PYTHON: "C:\\Python35"
-      TOXENV: "py35-ci-win"
+      TOXENV: "py35"
 
   SNAPSHOT_HOST:
     secure: NeTo57s2rJhCd/mjKHetXVxCFd3uhr8txnjnAXD1tUI=
@@ -22,18 +24,18 @@ install:
   - "python -m pip install --disable-pip-version-check -U pip"
   - "pip install -U tox"
 
-before_test:
-  - "tox -e lint"
-
 test_script:
-  - "tox"
+  - ps: "tox -- --cov netlib --cov mitmproxy --cov pathod | Select-String -NotMatch Cryptography_locking_cb"
 
 deploy_script:
   ps: |
-    if(($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true")) {
+    if(
+      ($Env:TOXENV -match "py27") -and
+      (($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
+      ) {
       pip install -U virtualenv
-      .\dev.bat
-      python .\release\rtool.py bdist
+      .\dev.ps1
+      cmd /c "python .\release\rtool.py bdist 2>&1"
       python .\release\rtool.py upload-snapshot --bdist
     }
 

.travis.yml

@@ -10,24 +10,31 @@ addons:
     packages:
       - libssl-dev
 
+env: 
+  global:
+    - CI_DEPS=codecov>=2.0.5
+    - CI_COMMANDS=codecov
+
 matrix:
   fast_finish: true
   include:
     - python: 2.7
-      env: TOXENV=py27-ci
+      env: TOXENV=py27
     - python: 2.7
-      env: TOXENV=py27-ci NO_ALPN=1
+      env: TOXENV=py27 NO_ALPN=1
     - python: 3.5
-      env: TOXENV=py35-ci
+      env: TOXENV=py35
     - python: 3.5
-      env: TOXENV=py35-ci NO_ALPN=1
+      env: TOXENV=py35 NO_ALPN=1
     - language: generic
-      env: TOXENV=py27-ci
       os: osx
       osx_image: xcode7.1
       git:
         depth: 9999999
-    - python: 2.7
+      env: TOXENV=py27
+    - python: 3.5
+      env: TOXENV=lint
+    - python: 3.5
       env: TOXENV=docs
   allow_failures:
     - python: pypy
@@ -42,10 +49,7 @@ install:
     fi
   - pip install tox
 
-before_script:
-  - "tox -e lint"
-
-script: tox
+script: set -o pipefail; tox -- --cov netlib --cov mitmproxy --cov pathod 2>&1 | grep -v Cryptography_locking_cb
 
 after_success:
   - |

README.rst

@@ -60,7 +60,7 @@ here_). Then do the following:
 
     git clone https://github.com/mitmproxy/mitmproxy.git
     cd mitmproxy
-    ./dev.sh
+    ./dev.sh  # powershell .\dev.ps1 on Windows
 
 
 The *dev* script will create a virtualenv environment in a directory called
@@ -74,7 +74,7 @@ mitmproxy test suite:
 
 .. code-block:: text
 
-    . venv/bin/activate # venv\Scripts\activate.bat on Windows
+    . venv/bin/activate  # venv\Scripts\activate on Windows
     py.test
 
 Note that the main executables for the project - ``mitmdump``, ``mitmproxy``,

dev.bat

@@ -1,16 +0,0 @@
-@echo off
-set VENV=.\venv
-
-virtualenv %VENV% --always-copy
-if %errorlevel% neq 0 exit /b %errorlevel%
-call %VENV%\Scripts\activate.bat
-if %errorlevel% neq 0 exit /b %errorlevel%
-python -m pip install --disable-pip-version-check -U pip
-if %errorlevel% neq 0 exit /b %errorlevel%
-pip install -r requirements.txt
-if %errorlevel% neq 0 exit /b %errorlevel%
-
-echo.
-echo * Created virtualenv environment in %VENV%.
-echo * Installed all dependencies into the virtualenv.
-echo * Activated virtualenv environment.

dev.ps1

@@ -0,0 +1,16 @@
+$ErrorActionPreference = "Stop"
+$VENV = ".\venv"
+
+virtualenv $VENV --always-copy
+& $VENV\Scripts\activate.ps1
+
+python -m pip install --disable-pip-version-check -U pip
+cmd /c "pip install -r requirements.txt 2>&1"
+
+echo @"
+
+  * Created virtualenv environment in $VENV.
+  * Installed all dependencies into the virtualenv.
+  * Activated virtualenv environment.
+
+"@

mitmproxy/script/reloader.py

@@ -1,16 +1,13 @@
 from __future__ import absolute_import, print_function, division
 
 import os
-import sys
 
 from watchdog.events import RegexMatchingEventHandler
 
-if sys.platform == 'darwin':  # pragma: no cover
 from watchdog.observers.polling import PollingObserver as Observer
-else:
-    from watchdog.observers import Observer
-# The OSX reloader in watchdog 0.8.3 breaks when unobserving paths.
-# We use the PollingObserver instead.
+# We occasionally have watchdog errors on Windows, Linux and Mac when using the native observers.
+# After reading through the watchdog source code and issue tracker,
+# we may want to replace this with a very simple implementation of our own.
 
 _observers = {}
 

requirements.txt

@@ -1,2 +1,3 @@
+https://maximilianhils.com/upload/2016-06/lxml-3.6.0-cp35-cp35m-win32.whl; sys_platform == 'win32' and python_version == '3.5'
 -e .[dev,examples,contentviews]
 -e ./release

setup.py

@@ -105,13 +105,14 @@ setup(
             "pytest-xdist>=1.14, <1.15",
             "sphinx>=1.3.5, <1.5",
             "sphinx-autobuild>=0.5.2, <0.7",
-            "sphinxcontrib-documentedlist>=0.3.0, <0.4",
+            "sphinxcontrib-documentedlist>=0.4.0, <0.5",
             "sphinx_rtd_theme>=0.1.9, <0.2",
         ],
         'contentviews': [
             "cssutils>=1.0.1, <1.1",
-            "protobuf>=2.6.1, <2.7",
-            "pyamf>=0.8.0, <0.9",
+            # TODO: Find Python 3 replacements
+            # "protobuf>=2.6.1, <2.7",
+            # "pyamf>=0.8.0, <0.9",
         ],
         'examples': [
             "beautifulsoup4>=4.4.1, <4.5",

test/mitmproxy/data/scripts/tcp_stream_modify.py

@@ -1,4 +1,4 @@
 def tcp_message(ctx, flow):
     message = flow.messages[-1]
     if not message.from_client:
-        message.content = message.content.replace("foo", "bar")
+        message.content = message.content.replace(b"foo", b"bar")

tox.ini

@@ -2,52 +2,27 @@
 envlist = py27, py35, docs, lint
 
 [testenv]
+usedevelop=True
 deps =
+  {env:CI_DEPS:}
   -rrequirements.txt
-  codecov>=2.0.5
-passenv = USERNAME HOME HOMEPATH LOCALAPPDATA CODECOV_TOKEN CI TRAVIS TRAVIS_* APPVEYOR APPVEYOR_*
+passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_*
 setenv =
-  PY3TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py
-
-[testenv:py27]
+  TESTS = test/
+  HOME = {envtmpdir}
 commands =
-  py.test -n 8 --timeout 60 []
+  py.test --timeout 60 {posargs} {env:TESTS}
+  {env:CI_COMMANDS:python -c ""}
 
 [testenv:py35]
-# remove bash & pipe & grep hack after cryptography ships with openssl 1.1.0
-whitelist_externals = bash
-commands =
-  bash -c 'set -o pipefail ; py.test -n 8 --timeout 60 {env:PY3TESTS:} 2>&1 | grep -v Cryptography_locking_cb'
-
-[testenv:py27-ci]
-commands =
-  py.test --cov netlib --cov mitmproxy --cov pathod --timeout 60
-  codecov -e TOXENV
-
-[testenv:py35-ci]
-# remove bash & pipe & grep hack after cryptography ships with openssl 1.1.0
-whitelist_externals = bash
-commands =
-  bash -c 'set -o pipefail ; py.test --cov netlib --cov mitmproxy --cov pathod --timeout 60 {env:PY3TESTS:} 2>&1 | grep -v Cryptography_locking_cb'
-  codecov -e TOXENV
-
-[testenv:py35-ci-win]
-deps =
-  https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl
-  codecov>=2.0.5
-  .[dev]
-commands =
-  py.test --cov netlib --cov mitmproxy --cov pathod --timeout 60 {env:PY3TESTS:}
-  codecov -e TOXENV
+setenv =
+  TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py
+  HOME = {envtmpdir}
 
 [testenv:docs]
-basepython = python2.7
-whitelist_externals = make
 changedir = docs
-setenv = SPHINXOPTS="-W"
-commands = make -e html
+commands = sphinx-build -W -b html -d {envtmpdir}/doctrees .  {envtmpdir}/html
 
 [testenv:lint]
-basepython = python2.7
 deps = flake8>=2.6.2, <3
 commands = flake8 --jobs 8 --count mitmproxy netlib pathod examples test