add test coverage protection

.appveyor.yml

@@ -28,7 +28,7 @@ install:
   - "pip install -U tox"
 
 test_script:
-  - ps: "tox -- --cov mitmproxy --cov pathod -v"
+  - ps: "tox -- --verbose --cov-report=term"
   - ps: |
       $Env:VERSION = $(python mitmproxy/version.py)
       $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""

.gitignore

@@ -20,3 +20,4 @@ bower_components
 sslkeylogfile.log
 .tox/
 .python-version
+coverage.xml

.travis.yml

@@ -63,7 +63,7 @@ install:
   - pip install tox
 
 script:
-  - tox -- --cov mitmproxy --cov pathod -v
+  - tox -- --verbose --cov-report=term
   - |
     if [[ $BDIST == "1" ]]
     then

test/conftest.py

@@ -1,5 +1,7 @@
+import os
 import pytest
 import OpenSSL
+
 import mitmproxy.net.tcp
 
 
@@ -12,3 +14,104 @@ requires_alpn = pytest.mark.skipif(
 def disable_alpn(monkeypatch):
     monkeypatch.setattr(mitmproxy.net.tcp, 'HAS_ALPN', False)
     monkeypatch.setattr(OpenSSL.SSL._lib, 'Cryptography_HAS_ALPN', False)
+
+
+enable_coverage = False
+coverage_values = []
+coverage_passed = False
+
+
+def pytest_addoption(parser):
+    parser.addoption('--full-cov',
+                     action='append',
+                     dest='full_cov',
+                     default=[],
+                     help="Require full test coverage of 100%% for this module/path/filename (multi-allowed). Default: none")
+
+    parser.addoption('--no-full-cov',
+                     action='append',
+                     dest='no_full_cov',
+                     default=[],
+                     help="Exclude file from a parent 100%% coverage requirement (multi-allowed). Default: none")
+
+
+def pytest_configure(config):
+    global enable_coverage
+    enable_coverage = (
+        len(config.getoption('file_or_dir')) == 0 and
+        len(config.getoption('full_cov')) > 0 and
+        config.pluginmanager.getplugin("_cov") is not None and
+        config.pluginmanager.getplugin("_cov").cov_controller is not None and
+        config.pluginmanager.getplugin("_cov").cov_controller.cov is not None
+    )
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtestloop(session):
+    global enable_coverage
+    global coverage_values
+    global coverage_passed
+
+    if not enable_coverage:
+        yield
+        return
+
+    cov = pytest.config.pluginmanager.getplugin("_cov").cov_controller.cov
+
+    if os.name == 'nt':
+        cov.exclude('pragma: windows no cover')
+
+    yield
+
+    coverage_values = dict([(name, 0) for name in pytest.config.option.full_cov])
+
+    prefix = os.getcwd()
+    excluded_files = [os.path.normpath(f) for f in pytest.config.option.no_full_cov]
+    measured_files = [os.path.normpath(os.path.relpath(f, prefix)) for f in cov.get_data().measured_files()]
+    measured_files = [f for f in measured_files if f not in excluded_files]
+
+    for name in pytest.config.option.full_cov:
+        files = [f for f in measured_files if f.startswith(os.path.normpath(name))]
+        try:
+            with open(os.devnull, 'w') as null:
+                coverage_values[name] = cov.report(files, ignore_errors=True, file=null)
+        except:
+            pass
+
+    if any(v < 100 for v in coverage_values.values()):
+        # make sure we get the EXIT_TESTSFAILED exit code
+        session.testsfailed += 1
+    else:
+        coverage_passed = True
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus):
+    global enable_coverage
+    global coverage_values
+    global coverage_passed
+
+    if not enable_coverage:
+        return
+
+    terminalreporter.write('\n')
+    if not coverage_passed:
+        markup = {'red': True, 'bold': True}
+        msg = "FAIL: Full test coverage not reached!\n"
+        terminalreporter.write(msg, **markup)
+
+        for name, value in coverage_values.items():
+            if value < 100:
+                markup = {'red': True, 'bold': True}
+            else:
+                markup = {'green': True}
+            msg = 'Coverage for {}: {:.2f}%\n'.format(name, value)
+            terminalreporter.write(msg, **markup)
+    else:
+        markup = {'green': True}
+        msg = 'SUCCESS: Full test coverage reached in modules and files:\n'
+        msg += '{}\n\n'.format('\n'.join(pytest.config.option.full_cov))
+        terminalreporter.write(msg, **markup)
+
+    msg = 'Excluded files:\n'
+    msg += '{}\n'.format('\n'.join(pytest.config.option.no_full_cov))
+    terminalreporter.write(msg)

tox.ini

@@ -11,7 +11,27 @@ passenv = CODECOV_TOKEN CI CI_* TRAVIS TRAVIS_* APPVEYOR APPVEYOR_* SNAPSHOT_* O
 setenv = HOME = {envtmpdir}
 commands =
   mitmdump --version
-  py.test --timeout 60 {posargs}
+  pytest --timeout 60 --cov-report='' --cov=mitmproxy --cov=pathod \
+    --full-cov=mitmproxy/addons/ \
+    --full-cov=mitmproxy/contentviews/ --no-full-cov=mitmproxy/contentviews/__init__.py --no-full-cov=mitmproxy/contentviews/protobuf.py --no-full-cov=mitmproxy/contentviews/wbxml.py --no-full-cov=mitmproxy/contentviews/xml_html.py \
+    --full-cov=mitmproxy/net/http/ --no-full-cov=mitmproxy/net/http/cookies.py --no-full-cov=mitmproxy/net/http/encoding.py --no-full-cov=mitmproxy/net/http/message.py --no-full-cov=mitmproxy/net/http/request.py --no-full-cov=mitmproxy/net/http/response.py --no-full-cov=mitmproxy/net/http/url.py \
+    --full-cov=mitmproxy/net/websockets/ \
+    --full-cov=mitmproxy/net/wsgi.py \
+    --full-cov=mitmproxy/proxy/__init__.py \
+    --full-cov=mitmproxy/proxy/modes/ --no-full-cov=mitmproxy/proxy/modes/socks_proxy.py \
+    --full-cov=mitmproxy/proxy/protocol/__init__.py \
+    --full-cov=mitmproxy/script/ \
+    --full-cov=mitmproxy/test/ --no-full-cov=mitmproxy/test/tutils.py \
+    --full-cov=mitmproxy/types/ --no-full-cov=mitmproxy/types/basethread.py \
+    --full-cov=mitmproxy/utils/ \
+    --full-cov=mitmproxy/__init__.py \
+    --full-cov=mitmproxy/addonmanager.py \
+    --full-cov=mitmproxy/ctx.py \
+    --full-cov=mitmproxy/exceptions.py \
+    --full-cov=mitmproxy/log.py \
+    --full-cov=mitmproxy/options.py \
+    --full-cov=pathod/ --no-full-cov=pathod/pathoc.py --no-full-cov=pathod/pathod.py --no-full-cov=pathod/test.py --no-full-cov=pathod/protocols/http2.py \
+    {posargs}
   {env:CI_COMMANDS:python -c ""}
 
 [testenv:docs]
@@ -24,12 +44,13 @@ commands =
   flake8 --jobs 8 --count mitmproxy pathod examples test release
   rstcheck README.rst
   mypy --silent-imports \
-    mitmproxy/addons \
+    mitmproxy/addons/ \
     mitmproxy/addonmanager.py \
     mitmproxy/proxy/protocol/ \
     mitmproxy/log.py \
-    mitmproxy/tools/dump.py mitmproxy/tools/web \
-    mitmproxy/contentviews
+    mitmproxy/tools/dump.py \
+    mitmproxy/tools/web/ \
+    mitmproxy/contentviews/
 
 [testenv:wheel]
 recreate = True