Merge pull request #4326 from mhils/py38

Update PyInstaller, require Python 3.8

.github/workflows/main.yml

@@ -51,10 +51,6 @@ jobs:
             py: 3.9
           - os: ubuntu-latest
             py: 3.8
-          - os: ubuntu-latest
-            py: 3.7
-          - os: ubuntu-latest
-            py: 3.6
     runs-on: ${{ matrix.os }}
     steps:
       - run: printenv
@@ -68,15 +64,6 @@ jobs:
         with:
           file: ./coverage.xml
           name: ${{ matrix.os }}
-  test-unsupported-python-version:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.5'
-      - run: pip install tox
-      - run: tox -e py35
   build-wheel:
     runs-on: ubuntu-latest
     env:
@@ -85,7 +72,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
+          python-version: '3.9'
       - run: pip install tox
       - run: tox -e cibuild -- build
       - uses: actions/upload-artifact@v2
@@ -107,7 +94,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.9'
       - if: matrix.os == 'windows-latest'
         uses: actions/cache@v1
         with:
@@ -153,7 +140,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
+          python-version: '3.9'
       - run: pip install tox
       - run: |
           wget https://github.com/gohugoio/hugo/releases/download/v0.70.0/hugo_extended_0.70.0_Linux-64bit.deb
@@ -173,7 +160,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.9'
       - run: pip install tox
       - uses: actions/download-artifact@v2
         with:
@@ -198,7 +185,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
+          python-version: '3.9'
       - uses: actions/download-artifact@v2
         with:
           path: release/dist

CHANGELOG.rst

@@ -4,13 +4,16 @@ Release History
 Unreleased: mitmproxy next
 ==========================
 
+* Mitmproxy now requires Python 3.8 or above.
 * Fix query parameters in asgiapp addon (@jpstotz)
 * Fix command history failing on file IO errors (@Kriechi)
 * Deprecation of pathod and pathoc tools and modules. Future releases might not contain them! (@Kriechi)
 * Addon to suppress unwanted error messages sent by mitmproxy. (@anneborcherding)
 * Updated imports and styles for web scanner helper addons. (@anneborcherding)
 * Inform when underscore-formatted options are used in client arg. (@jrblixt)
+* Binaries are now built with Python 3.9 (@mhils)
 * Fixed the web UI showing blank page on clicking details tab when server address is missing (@samhita-sopho)
+
 * --- TODO: add new PRs above this line ---
 
 * ... and various other fixes, documentation improvements, dependency version bumps, etc.

dev.ps1

@@ -1,10 +1,5 @@
 $ErrorActionPreference = "Stop"
 
-$pyver = python --version
-if($pyver -notmatch "3\.[6-9]") {
-    Write-Warning "Unexpected Python version, expected Python 3.6 or above: $pyver"
-}
-
 python -m venv .\venv --copies
 & .\venv\Scripts\activate.ps1
 

mitmproxy/master.py

@@ -90,7 +90,7 @@ class Master:
             if not self.should_exit.is_set():  # pragma: no cover
                 self.shutdown()
             loop = asyncio.get_event_loop()
-            tasks = asyncio.all_tasks(loop) if sys.version_info >= (3, 7) else asyncio.Task.all_tasks(loop)
+            tasks = asyncio.all_tasks(loop)
             for p in tasks:
                 p.cancel()
             loop.close()

mitmproxy/net/http/url.py

@@ -56,7 +56,7 @@ def parse(url):
         if isinstance(parsed, urllib.parse.ParseResult):
             parsed = parsed.encode("ascii")
 
-    port = parsed.port  # Returns None if port number invalid in Py3.5. Will throw ValueError in Py3.6
+    port = parsed.port
     if not port:
         port = 443 if parsed.scheme == b"https" else 80
 

mitmproxy/net/tcp.py

@@ -20,7 +20,7 @@ from mitmproxy.coretypes import basethread
 socket_fileobject = socket.SocketIO
 
 # workaround for https://bugs.python.org/issue29515
-# Python 3.6 for Windows is missing a constant
+# Python 3.8 for Windows is missing a constant, fixed in 3.9
 IPPROTO_IPV6 = getattr(socket, "IPPROTO_IPV6", 41)
 
 

mitmproxy/tools/_main.py

@@ -1,174 +0,0 @@
-"""
-This file contains python3.6+ syntax!
-Feel free to import and use whatever new package you deem necessary.
-"""
-
-import os
-import sys
-import asyncio
-import argparse
-import signal
-import typing
-
-from mitmproxy.tools import cmdline
-from mitmproxy import exceptions, master
-from mitmproxy import options
-from mitmproxy import optmanager
-from mitmproxy import proxy
-from mitmproxy.utils import compat, debug, arg_check
-
-
-def assert_utf8_env():
-    spec = ""
-    for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
-        spec += os.environ.get(i, "").lower()
-    if "utf" not in spec:
-        print(
-            "Error: mitmproxy requires a UTF console environment.",
-            file=sys.stderr
-        )
-        print(
-            "Set your LANG environment variable to something like en_US.UTF-8",
-            file=sys.stderr
-        )
-        sys.exit(1)
-
-
-def process_options(parser, opts, args):
-    if args.version:
-        print(debug.dump_system_info())
-        sys.exit(0)
-    if args.quiet or args.options or args.commands:
-        # also reduce log verbosity if --options or --commands is passed,
-        # we don't want log messages from regular startup then.
-        args.termlog_verbosity = 'error'
-        args.flow_detail = 0
-    if args.verbose:
-        args.termlog_verbosity = 'debug'
-        args.flow_detail = 2
-
-    adict = {}
-    for n in dir(args):
-        if n in opts:
-            adict[n] = getattr(args, n)
-    opts.merge(adict)
-
-    return proxy.config.ProxyConfig(opts)
-
-
-def run(
-        master_cls: typing.Type[master.Master],
-        make_parser: typing.Callable[[options.Options], argparse.ArgumentParser],
-        arguments: typing.Sequence[str],
-        extra: typing.Callable[[typing.Any], dict] = None
-) -> master.Master:  # pragma: no cover
-    """
-        extra: Extra argument processing callable which returns a dict of
-        options.
-    """
-    debug.register_info_dumpers()
-
-    opts = options.Options()
-    master = master_cls(opts)
-
-    parser = make_parser(opts)
-
-    # To make migration from 2.x to 3.0 bearable.
-    if "-R" in sys.argv and sys.argv[sys.argv.index("-R") + 1].startswith("http"):
-        print("To use mitmproxy in reverse mode please use --mode reverse:SPEC instead")
-
-    try:
-        args = parser.parse_args(arguments)
-    except SystemExit:
-        arg_check.check()
-        sys.exit(1)
-
-    try:
-        opts.set(*args.setoptions, defer=True)
-        optmanager.load_paths(
-            opts,
-            os.path.join(opts.confdir, "config.yaml"),
-            os.path.join(opts.confdir, "config.yml"),
-        )
-        pconf = process_options(parser, opts, args)
-        server: typing.Any = None
-        if pconf.options.server and not compat.new_proxy_core:  # new core initializes itself as an addon
-            try:
-                server = proxy.server.ProxyServer(pconf)
-            except exceptions.ServerException as v:
-                print(str(v), file=sys.stderr)
-                sys.exit(1)
-        else:
-            server = proxy.server.DummyServer(pconf)
-
-        master.server = server
-        if args.options:
-            print(optmanager.dump_defaults(opts))
-            sys.exit(0)
-        if args.commands:
-            master.commands.dump()
-            sys.exit(0)
-        if extra:
-            if(args.filter_args):
-                master.log.info(f"Only processing flows that match \"{' & '.join(args.filter_args)}\"")
-            opts.update(**extra(args))
-
-        loop = asyncio.get_event_loop()
-        try:
-            loop.add_signal_handler(signal.SIGINT, getattr(master, "prompt_for_exit", master.shutdown))
-            loop.add_signal_handler(signal.SIGTERM, master.shutdown)
-        except NotImplementedError:
-            # Not supported on Windows
-            pass
-
-        # Make sure that we catch KeyboardInterrupts on Windows.
-        # https://stackoverflow.com/a/36925722/934719
-        if os.name == "nt":
-            async def wakeup():
-                while True:
-                    await asyncio.sleep(0.2)
-            asyncio.ensure_future(wakeup())
-
-        master.run()
-    except exceptions.OptionsError as e:
-        print("{}: {}".format(sys.argv[0], e), file=sys.stderr)
-        sys.exit(1)
-    except (KeyboardInterrupt, RuntimeError):
-        pass
-    return master
-
-
-def mitmproxy(args=None) -> typing.Optional[int]:  # pragma: no cover
-    if os.name == "nt":
-        print("Error: mitmproxy's console interface is not supported on Windows. "
-              "You can run mitmdump or mitmweb instead.", file=sys.stderr)
-        return 1
-    assert_utf8_env()
-    from mitmproxy.tools import console
-    run(console.master.ConsoleMaster, cmdline.mitmproxy, args)
-    return None
-
-
-def mitmdump(args=None) -> typing.Optional[int]:  # pragma: no cover
-    from mitmproxy.tools import dump
-
-    def extra(args):
-        if args.filter_args:
-            v = " ".join(args.filter_args)
-            return dict(
-                save_stream_filter=v,
-                readfile_filter=v,
-                dumper_filter=v,
-            )
-        return {}
-
-    m = run(dump.DumpMaster, cmdline.mitmdump, args, extra)
-    if m and m.errorcheck.has_errored:  # type: ignore
-        return 1
-    return None
-
-
-def mitmweb(args=None) -> typing.Optional[int]:  # pragma: no cover
-    from mitmproxy.tools import web
-    run(web.master.WebMaster, cmdline.mitmweb, args)
-    return None

mitmproxy/tools/main.py

@@ -1,22 +1,169 @@
-"""
+import os
-This file must be kept in a python2.7 and python3.5 compatible syntax!
-DO NOT use type annotations or other python3.6-only features that makes this file unparsable by older interpreters!
-"""
-
-from __future__ import print_function  # this is here for the version check to work on Python 2.
-
 import sys
+import asyncio
+import argparse
+import signal
+import typing
 
-if sys.version_info < (3, 6):
+from mitmproxy.tools import cmdline
-    # This must be before any mitmproxy imports, as they already break!
+from mitmproxy import exceptions, master
-    # Keep all other imports below with the 'noqa' magic comment.
+from mitmproxy import options
-    print("#" * 76, file=sys.stderr)
+from mitmproxy import optmanager
-    print("# mitmproxy requires Python 3.6 or higher!                                  #", file=sys.stderr)
+from mitmproxy import proxy
-    print("#" + " " * 74 + "#", file=sys.stderr)
+from mitmproxy.utils import compat, debug, arg_check
-    print("# Please upgrade your Python interpreter or use our mitmproxy binaries from #", file=sys.stderr)
+
-    print("# https://mitmproxy.org. If your operating system does not include the      #", file=sys.stderr)
+
-    print("# required Python version, you can try using pyenv or similar tools.        #", file=sys.stderr)
+def assert_utf8_env():
-    print("#" * 76, file=sys.stderr)
+    spec = ""
-    sys.exit(1)
+    for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
-else:
+        spec += os.environ.get(i, "").lower()
-    from ._main import *  # noqa
+    if "utf" not in spec:
+        print(
+            "Error: mitmproxy requires a UTF console environment.",
+            file=sys.stderr
+        )
+        print(
+            "Set your LANG environment variable to something like en_US.UTF-8",
+            file=sys.stderr
+        )
+        sys.exit(1)
+
+
+def process_options(parser, opts, args):
+    if args.version:
+        print(debug.dump_system_info())
+        sys.exit(0)
+    if args.quiet or args.options or args.commands:
+        # also reduce log verbosity if --options or --commands is passed,
+        # we don't want log messages from regular startup then.
+        args.termlog_verbosity = 'error'
+        args.flow_detail = 0
+    if args.verbose:
+        args.termlog_verbosity = 'debug'
+        args.flow_detail = 2
+
+    adict = {}
+    for n in dir(args):
+        if n in opts:
+            adict[n] = getattr(args, n)
+    opts.merge(adict)
+
+    return proxy.config.ProxyConfig(opts)
+
+
+def run(
+        master_cls: typing.Type[master.Master],
+        make_parser: typing.Callable[[options.Options], argparse.ArgumentParser],
+        arguments: typing.Sequence[str],
+        extra: typing.Callable[[typing.Any], dict] = None
+) -> master.Master:  # pragma: no cover
+    """
+        extra: Extra argument processing callable which returns a dict of
+        options.
+    """
+    debug.register_info_dumpers()
+
+    opts = options.Options()
+    master = master_cls(opts)
+
+    parser = make_parser(opts)
+
+    # To make migration from 2.x to 3.0 bearable.
+    if "-R" in sys.argv and sys.argv[sys.argv.index("-R") + 1].startswith("http"):
+        print("To use mitmproxy in reverse mode please use --mode reverse:SPEC instead")
+
+    try:
+        args = parser.parse_args(arguments)
+    except SystemExit:
+        arg_check.check()
+        sys.exit(1)
+
+    try:
+        opts.set(*args.setoptions, defer=True)
+        optmanager.load_paths(
+            opts,
+            os.path.join(opts.confdir, "config.yaml"),
+            os.path.join(opts.confdir, "config.yml"),
+        )
+        pconf = process_options(parser, opts, args)
+        server: typing.Any = None
+        if pconf.options.server and not compat.new_proxy_core:  # new core initializes itself as an addon
+            try:
+                server = proxy.server.ProxyServer(pconf)
+            except exceptions.ServerException as v:
+                print(str(v), file=sys.stderr)
+                sys.exit(1)
+        else:
+            server = proxy.server.DummyServer(pconf)
+
+        master.server = server
+        if args.options:
+            print(optmanager.dump_defaults(opts))
+            sys.exit(0)
+        if args.commands:
+            master.commands.dump()
+            sys.exit(0)
+        if extra:
+            if(args.filter_args):
+                master.log.info(f"Only processing flows that match \"{' & '.join(args.filter_args)}\"")
+            opts.update(**extra(args))
+
+        loop = asyncio.get_event_loop()
+        try:
+            loop.add_signal_handler(signal.SIGINT, getattr(master, "prompt_for_exit", master.shutdown))
+            loop.add_signal_handler(signal.SIGTERM, master.shutdown)
+        except NotImplementedError:
+            # Not supported on Windows
+            pass
+
+        # Make sure that we catch KeyboardInterrupts on Windows.
+        # https://stackoverflow.com/a/36925722/934719
+        if os.name == "nt":
+            async def wakeup():
+                while True:
+                    await asyncio.sleep(0.2)
+            asyncio.ensure_future(wakeup())
+
+        master.run()
+    except exceptions.OptionsError as e:
+        print("{}: {}".format(sys.argv[0], e), file=sys.stderr)
+        sys.exit(1)
+    except (KeyboardInterrupt, RuntimeError):
+        pass
+    return master
+
+
+def mitmproxy(args=None) -> typing.Optional[int]:  # pragma: no cover
+    if os.name == "nt":
+        print("Error: mitmproxy's console interface is not supported on Windows. "
+              "You can run mitmdump or mitmweb instead.", file=sys.stderr)
+        return 1
+    assert_utf8_env()
+    from mitmproxy.tools import console
+    run(console.master.ConsoleMaster, cmdline.mitmproxy, args)
+    return None
+
+
+def mitmdump(args=None) -> typing.Optional[int]:  # pragma: no cover
+    from mitmproxy.tools import dump
+
+    def extra(args):
+        if args.filter_args:
+            v = " ".join(args.filter_args)
+            return dict(
+                save_stream_filter=v,
+                readfile_filter=v,
+                dumper_filter=v,
+            )
+        return {}
+
+    m = run(dump.DumpMaster, cmdline.mitmdump, args, extra)
+    if m and m.errorcheck.has_errored:  # type: ignore
+        return 1
+    return None
+
+
+def mitmweb(args=None) -> typing.Optional[int]:  # pragma: no cover
+    from mitmproxy.tools import web
+    run(web.master.WebMaster, cmdline.mitmweb, args)
+    return None

mitmproxy/utils/debug.py

@@ -83,8 +83,6 @@ def dump_info(signal=None, frame=None, file=sys.stdout, testing=False):  # pragm
             print(i[1], i[0])
 
         try:
-            if sys.version_info < (3, 8):
-                raise RuntimeError
             asyncio.get_running_loop()
         except RuntimeError:
             pass

setup.py

@@ -34,8 +34,6 @@ setup(
         "Operating System :: POSIX",
         "Operating System :: Microsoft :: Windows",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: Implementation :: CPython",
@@ -65,6 +63,7 @@ setup(
             "pathoc = pathod.pathoc_cmdline:go_pathoc"
         ]
     },
+    python_requires='>=3.8',
     # https://packaging.python.org/en/latest/requirements/#install-requires
     # It is not considered best practice to use install_requires to pin dependencies to specific versions.
     install_requires=[
@@ -75,8 +74,8 @@ setup(
         "click>=7.0,<8",
         "cryptography>=3.2,<3.3",
         "flask>=1.1.1,<1.2",
-        "h2>=4.0,<5; python_version>='3.6.0'",  # python_version only needed to make "py36+ required" message work
+        "h2>=4.0,<5",
-        "hyperframe>=6.0,<7; python_version>='3.6.0'",  # python_version only needed to make "py36+ required" message work
+        "hyperframe>=6.0,<7",
         "kaitaistruct>=0.7,<0.10",
         "ldap3>=2.8,<2.9",
         "msgpack>=1.0.0, <1.1.0",
@@ -98,9 +97,6 @@ setup(
         ':sys_platform == "win32"': [
             "pydivert>=2.0.3,<2.2",
         ],
-        ':python_version == "3.6"': [
-            "dataclasses>=0.7",
-        ],
         'dev': [
             "hypothesis>=5.8,<6",
             "parver>=0.1,<2.0",

test/mitmproxy/addons/test_readfile.py

@@ -1,6 +1,5 @@
 import asyncio
 import io
-import sys
 
 import pytest
 from unittest import mock
@@ -47,7 +46,6 @@ class TestReadFile:
             with pytest.raises(Exception, match="Invalid readfile filter"):
                 tctx.configure(rf, readfile_filter="~~")
 
-    @pytest.mark.skipif(sys.version_info < (3, 8), reason='requires Python 3.8 or higher')
     @pytest.mark.asyncio
     async def test_read(self, tmpdir, data, corrupt_data):
         rf = readfile.ReadFile()
@@ -95,7 +93,6 @@ class TestReadFile:
 
 
 class TestReadFileStdin:
-    @pytest.mark.skipif(sys.version_info < (3, 8), reason='requires Python 3.8 or higher')
     @mock.patch('sys.stdin')
     @pytest.mark.asyncio
     async def test_stdin(self, stdin, data, corrupt_data):
@@ -111,7 +108,6 @@ class TestReadFileStdin:
                 with pytest.raises(exceptions.FlowReadException):
                     await rf.load_flows(stdin.buffer)
 
-    @pytest.mark.skipif(sys.version_info < (3, 8), reason='requires Python 3.8 or higher')
     @pytest.mark.asyncio
     async def test_normal(self, tmpdir, data):
         rf = readfile.ReadFileStdin()

test/mitmproxy/net/http/test_url.py

@@ -1,7 +1,6 @@
 from typing import AnyStr
 
 import pytest
-import sys
 
 from mitmproxy.net.http import url
 from mitmproxy.net.http.url import parse_authority
@@ -62,7 +61,6 @@ def test_ascii_check():
                         b'%BD%E7%8C%AB%E6%B0%93%E7%8C%AB%E6%B0%93'
 
 
-@pytest.mark.skipif(sys.version_info < (3, 6), reason='requires Python 3.6 or higher')
 def test_parse_port_range():
     # Port out of range
     with pytest.raises(ValueError):

test/mitmproxy/utils/test_asyncio_utils.py

@@ -1,5 +1,4 @@
 import asyncio
-import sys
 
 import pytest
 
@@ -15,7 +14,6 @@ async def ttask():
 
 
 @pytest.mark.asyncio
-@pytest.mark.skipif(sys.version_info < (3, 8), reason="requires Python 3.8")
 async def test_simple():
     task = asyncio_utils.create_task(
         ttask(),
@@ -30,7 +28,6 @@ async def test_simple():
     assert task.cancelled()
 
 
-@pytest.mark.skipif(sys.version_info < (3, 8), reason="requires Python 3.8")
 def test_closed_loop():
     # Crude test for line coverage.
     # This should eventually go, see the description in asyncio_utils.create_task for details.
@@ -38,9 +35,11 @@ def test_closed_loop():
         ttask(),
         name="ttask",
     )
+    t = ttask()
     with pytest.raises(RuntimeError):
         asyncio_utils.create_task(
-            ttask(),
+            t,
             name="ttask",
             ignore_closed_loop=False,
         )
+    t.close()  # suppress "not awaited" warning

test/mitmproxy/utils/test_debug.py

@@ -24,8 +24,7 @@ def test_dump_info():
 async def test_dump_info_async():
     cs = io.StringIO()
     debug.dump_info(None, None, file=cs, testing=True)
-    if sys.version_info >= (3, 8):
+    assert "Tasks" in cs.getvalue()
-        assert "Tasks" in cs.getvalue()
 
 
 def test_dump_stacks():

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py35, py36, py37, py38, py39, flake8, filename_matching, mypy, individual_coverage, docs
+envlist = py38, py39, flake8, filename_matching, mypy, individual_coverage, docs
 skipsdist = True
 toxworkdir={env:TOX_WORK_DIR:.tox}
 
@@ -14,14 +14,6 @@ commands =
     --full-cov=mitmproxy/ --full-cov=pathod/ \
     {posargs}
 
-[testenv:py35]
-whitelist_externals =
-  bash
-deps =
-  -rrequirements.txt
-commands =
-  bash -c "mitmdump --version 2>&1 | grep 'mitmproxy requires Python 3.6'"
-
 [testenv:flake8]
 deps = flake8==3.8.4
 commands =
@@ -51,7 +43,7 @@ commands =
 passenv = CI_* GITHUB_* AWS_* TWINE_* DOCKER_*
 deps =
   -rrequirements.txt
-  pyinstaller==4.0
+  pyinstaller==4.1
   twine==3.2.0
   awscli
 commands =