Merge pull request #3767 from mitmproxy/fix-ci

re-add missing CI steps and fix linting

.github/workflows/main.yml

@@ -2,10 +2,6 @@ name: CI
 
 on: [push, pull_request]
 
-env:
-  # Codecov
-  CODECOV_TOKEN: "0409bdfd-57a4-477d-a8af-f6172ef431d3"
-
 jobs:
   lint-pr:
     if: github.event_name == 'pull_request'
@@ -14,21 +10,34 @@ jobs:
       - uses: actions/checkout@v1
       - uses: TrueBrain/actions-flake8@v1.2
   lint-local:
-    # do not use external action when secrets are exposed.
     if: github.event_name == 'push'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
       - uses: actions/setup-python@v1
-      - run: pip install flake8
-      - run: flake8 mitmproxy pathod examples test release
+      - run: pip install tox
+      - run: tox -e flake8
+  filename-matching:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+      - run: pip install tox
+      - run: tox -e filename_matching
   mypy:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
       - uses: actions/setup-python@v1
-      - run: pip install mypy==0.770
-      - run: mypy .
+      - run: pip install tox
+      - run: tox -e mypy
+  individual-coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+      - run: pip install tox
+      - run: tox -e individual_coverage
   test:
     strategy:
       fail-fast: false
@@ -43,19 +52,11 @@ jobs:
           python-version: '3.8'
       - run: pip install tox
       - run: tox -e py38
-      # codecov's GitHub action only supports Linux. https://github.com/codecov/codecov-action/issues/7
-      # codecov's Python uploader has no github actions support yet. https://github.com/codecov/codecov-python/pull/214
-      - name: Extract branch name  # https://stackoverflow.com/a/58035262/934719
-        shell: bash
-        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: extract_branch
-      - run: pip install codecov
-      - run: >
-          codecov -f coverage.xml
-          --name python-${{ matrix.os }}
-          --commit ${{ github.sha }}
-          --slug ${{ github.repository }}
-          --branch ${{ steps.extract_branch.outputs.branch }}
+      - uses: codecov/codecov-action@v1
+        with:
+          file: ./coverage.xml
+          name: ${{ matrix.os }}
+          fail_ci_if_error: true
   test-py35:
     runs-on: ubuntu-latest
     steps:
@@ -127,7 +128,11 @@ jobs:
         run: yarn
       - working-directory: ./web
         run: npm test
-      - run: bash <(curl -s https://codecov.io/bash)
+      - uses: codecov/codecov-action@v1
+        with:
+          file: ./web/coverage/coverage-final.json
+          name: web
+          fail_ci_if_error: true
 
   docs:
     env:

README.rst

@@ -95,12 +95,13 @@ Testing
 -------
 
 If you've followed the procedure above, you already have all the development
-requirements installed, and you can run the full test suite with tox_:
+requirements installed, and you can run the basic test suite with tox_:
 
 .. code-block:: bash
 
-    tox -e py    # runs Python tests
-    tox -e lint  # checks code style
+    tox -e py      # runs Python tests
+
+Our CI system has additional tox environments that are run on every pull request and branch on GitHub.
 
 For speedier testing, we recommend you run `pytest`_ directly on individual test files or folders:
 
@@ -109,7 +110,7 @@ For speedier testing, we recommend you run `pytest`_ directly on individual test
     cd test/mitmproxy/addons
     pytest --cov mitmproxy.addons.anticache --cov-report term-missing --looponfail test_anticache.py
 
-Pytest does not check the code style, so you want to run ``tox -e lint`` again before committing.
+Pytest does not check the code style, so you want to run ``tox -e flake8`` again before committing.
 
 Please ensure that all patches are accompanied by matching changes in the test
 suite. The project tries to maintain 100% test coverage and enforces this strictly for some parts of the codebase.
@@ -138,11 +139,12 @@ good reason not to.
 
 This is automatically enforced on every PR. If we detect a linting error, the
 PR checks will fail and block merging. You can run our lint checks yourself
-with the following command:
+with the following commands:
 
 .. code-block:: bash
 
-    tox -e lint
+    tox -e flake8
+    tox -e mypy    # checks static types
 
 
 .. |mitmproxy_site| image:: https://shields.mitmproxy.org/badge/https%3A%2F%2F-mitmproxy.org-blue.svg

mitmproxy/addons/command_history.py

@@ -27,7 +27,7 @@ class CommandHistory:
     def running(self):
         # FIXME: We have a weird bug where the contract for configure is not followed and it is never called with
         # confdir or command_history as updated.
-        self.configure("command_history")
+        self.configure("command_history")  # pragma: no cover
 
     def configure(self, updated):
         if "command_history" in updated or "confdir" in updated:
@@ -36,9 +36,9 @@ class CommandHistory:
                 self.set_filter('')
 
     def done(self):
-        if ctx.options.command_history and len(self.history) > self.VACUUM_SIZE:
+        if ctx.options.command_history and len(self.history) >= self.VACUUM_SIZE:
             # vacuum history so that it doesn't grow indefinitely.
-            history_str = "\n".join(self.history[-self.VACUUM_SIZE / 2:]) + "\n"
+            history_str = "\n".join(self.history[-self.VACUUM_SIZE // 2:]) + "\n"
             self.history_file.write_text(history_str)
 
     @command.command("commands.history.add")

mitmproxy/net/http/http1/read.py

@@ -331,7 +331,8 @@ def _read_headers(rfile):
     while True:
         line = rfile.readline()
         if not line or line == b"\r\n" or line == b"\n":
-            break
+            # we do have coverage of this, but coverage.py does not detect it.
+            break  # pragma: no cover
         if line[0] in b" \t":
             if not ret:
                 raise exceptions.HttpSyntaxException("Invalid headers")

setup.py

@@ -92,9 +92,8 @@ setup(
         ],
         'dev': [
             "asynctest>=0.12.0",
-            "flake8>=3.7.8,<3.8",
             "Flask>=1.0,<1.2",
-            "mypy>=0.740,<0.771",
+            "hypothesis>=5.8,<5.9",
             "parver>=0.1,<2.0",
             "pytest-asyncio>=0.10.0,<0.11",
             "pytest-cov>=2.7.1,<3",
@@ -103,7 +102,6 @@ setup(
             "pytest>=5.1.3,<6",
             "requests>=2.9.1,<3",
             "tox>=3.5,<3.15",
-            "rstcheck>=2.2,<4.0",
         ],
         'examples': [
             "beautifulsoup4>=4.4.1,<4.9"

test/mitmproxy/addons/test_command_history.py

@@ -5,15 +5,22 @@ from mitmproxy.test import taddons
 
 
 class TestCommandHistory:
-    def test_load_from_file(self, tmpdir):
-        commands = ['cmd1', 'cmd2', 'cmd3']
-        with open(tmpdir.join('command_history'), 'w') as f:
+    def test_load_and_save(self, tmpdir):
+        history_file = tmpdir.join('command_history')
+        commands = ["cmd1", "cmd2", "cmd3"]
+        with open(history_file, 'w') as f:
             f.write("\n".join(commands))
 
         ch = command_history.CommandHistory()
+        ch.VACUUM_SIZE = 4
         with taddons.context(ch) as tctx:
             tctx.options.confdir = str(tmpdir)
             assert ch.history == commands
+            ch.add_command("cmd4")
+            ch.done()
+
+        with open(history_file, "r") as f:
+            assert f.read() == "cmd3\ncmd4\n"
 
     def test_add_command(self):
         history = command_history.CommandHistory()

test/mitmproxy/net/http/http1/test_read.py

@@ -161,7 +161,8 @@ def test_connection_close():
 def test_expected_http_body_size():
     # Expect: 100-continue
     assert expected_http_body_size(
-        treq(headers=Headers(expect="100-continue", content_length="42"))
+        treq(headers=Headers(expect="100-continue", content_length="42")),
+        expect_continue_as_0=True
     ) == 0
     # Expect: 100-continue
     assert expected_http_body_size(

test/mitmproxy/test_command_lexer.py

@@ -1,5 +1,7 @@
 import pyparsing
 import pytest
+from hypothesis import given, example
+from hypothesis.strategies import text
 
 from mitmproxy import command_lexer
 
@@ -36,3 +38,14 @@ def test_partial_quoted_string(test_input, valid):
 )
 def test_expr(test_input, expected):
     assert list(command_lexer.expr.parseString(test_input, parseAll=True)) == expected
+
+
+@given(text())
+def test_quote_unquote_cycle(s):
+    assert command_lexer.unquote(command_lexer.quote(s)) == s
+
+
+@given(text())
+@example("'foo\\'")
+def test_unquote_never_fails(s):
+    command_lexer.unquote(s)

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py35, py36, py37, lint, individual_coverage, wheeltest, docs
+envlist = py35, py36, py37, flake8, filename_matching, mypy, individual_coverage, docs
 skipsdist = True
 toxworkdir={env:TOX_WORK_DIR:.tox}
 
@@ -22,14 +22,25 @@ deps =
 commands =
   bash -c "mitmdump --version 2>&1 | grep 'mitmproxy requires Python 3.6'"
 
-[testenv:lint]
+[testenv:flake8]
+deps = flake8>=3.7.8,<3.8
 commands =
-  mitmdump --version
   flake8 --jobs 8 mitmproxy pathod examples test release
+
+[testenv:filename_matching]
+commands =
   python ./test/filename_matching.py
-  rstcheck README.rst
+
+[testenv:mypy]
+deps = mypy>=0.761,<0.762
+commands =
   mypy .
 
+[testenv:rstcheck]
+deps = rstcheck>=2.2,<4.0
+commands =
+  rstcheck README.rst
+
 [testenv:individual_coverage]
 deps =
   -rrequirements.txt