Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-26 18:18:25 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4326 from mhils/py38

Browse Source 
Update PyInstaller, require Python 3.8
This commit is contained in:
Maximilian Hils 2020-12-07 23:06:38 +01:00 committed by GitHub
commit d6585965de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 187 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
.github/workflows/main.yml vendored
View File

@ -51,10 +51,6 @@ jobs:
py: 3.9
- os: ubuntu-latest
py: 3.8
- os: ubuntu-latest
py: 3.7
- os: ubuntu-latest
py: 3.6
runs-on: ${{ matrix.os }}
steps:
- run: printenv
@ -68,15 +64,6 @@ jobs:
with:
file: ./coverage.xml
name: ${{ matrix.os }}
test-unsupported-python-version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v1
- uses: actions/setup-python@v2
with:
python-version: '3.5'
- run: pip install tox
- run: tox -e py35
build-wheel:
runs-on: ubuntu-latest
env:
@ -85,7 +72,7 @@ jobs:
- uses: actions/checkout@v1
- uses: actions/setup-python@v2
with:
python-version: '3.8'
python-version: '3.9'
- run: pip install tox
- run: tox -e cibuild -- build
- uses: actions/upload-artifact@v2
@ -107,7 +94,7 @@ jobs:
- uses: actions/checkout@v1
- uses: actions/setup-python@v2
with:
python-version: '3.7'
python-version: '3.9'
- if: matrix.os == 'windows-latest'
uses: actions/cache@v1
with:
@ -153,7 +140,7 @@ jobs:
- uses: actions/checkout@v1
- uses: actions/setup-python@v2
with:
python-version: '3.8'
python-version: '3.9'
- run: pip install tox
- run: |
wget https://github.com/gohugoio/hugo/releases/download/v0.70.0/hugo_extended_0.70.0_Linux-64bit.deb
@ -173,7 +160,7 @@ jobs:
- uses: actions/checkout@v1
- uses: actions/setup-python@v2
with:
python-version: '3.7'
python-version: '3.9'
- run: pip install tox
- uses: actions/download-artifact@v2
with:
@ -198,7 +185,7 @@ jobs:
- uses: actions/checkout@v1
- uses: actions/setup-python@v2
with:
python-version: '3.8'
python-version: '3.9'
- uses: actions/download-artifact@v2
with:
path: release/dist

3
CHANGELOG.rst
View File

@ -4,13 +4,16 @@ Release History
Unreleased: mitmproxy next
==========================
* Mitmproxy now requires Python 3.8 or above.
* Fix query parameters in asgiapp addon (@jpstotz)
* Fix command history failing on file IO errors (@Kriechi)
* Deprecation of pathod and pathoc tools and modules. Future releases might not contain them! (@Kriechi)
* Addon to suppress unwanted error messages sent by mitmproxy. (@anneborcherding)
* Updated imports and styles for web scanner helper addons. (@anneborcherding)
* Inform when underscore-formatted options are used in client arg. (@jrblixt)
* Binaries are now built with Python 3.9 (@mhils)
* Fixed the web UI showing blank page on clicking details tab when server address is missing (@samhita-sopho)
* --- TODO: add new PRs above this line ---
* ... and various other fixes, documentation improvements, dependency version bumps, etc.

5
dev.ps1
View File

@ -1,10 +1,5 @@
$ErrorActionPreference = "Stop"
$pyver = python --version
if($pyver -notmatch "3\.[6-9]") {
Write-Warning "Unexpected Python version, expected Python 3.6 or above: $pyver"
}
python -m venv .\venv --copies
& .\venv\Scripts\activate.ps1

2
mitmproxy/master.py
View File

@ -90,7 +90,7 @@ class Master:
if not self.should_exit.is_set(): # pragma: no cover
self.shutdown()
loop = asyncio.get_event_loop()
tasks = asyncio.all_tasks(loop) if sys.version_info >= (3, 7) else asyncio.Task.all_tasks(loop)
tasks = asyncio.all_tasks(loop)
for p in tasks:
p.cancel()
loop.close()

2
mitmproxy/net/http/url.py
View File

@ -56,7 +56,7 @@ def parse(url):
if isinstance(parsed, urllib.parse.ParseResult):
parsed = parsed.encode("ascii")
port = parsed.port # Returns None if port number invalid in Py3.5. Will throw ValueError in Py3.6
port = parsed.port
if not port:
port = 443 if parsed.scheme == b"https" else 80

2
mitmproxy/net/tcp.py
View File

@ -20,7 +20,7 @@ from mitmproxy.coretypes import basethread
socket_fileobject = socket.SocketIO
# workaround for https://bugs.python.org/issue29515
# Python 3.6 for Windows is missing a constant
# Python 3.8 for Windows is missing a constant, fixed in 3.9
IPPROTO_IPV6 = getattr(socket, "IPPROTO_IPV6", 41)

174
mitmproxy/tools/_main.py
View File

@ -1,174 +0,0 @@
"""
This file contains python3.6+ syntax!
Feel free to import and use whatever new package you deem necessary.
"""
import os
import sys
import asyncio
import argparse
import signal
import typing
from mitmproxy.tools import cmdline
from mitmproxy import exceptions, master
from mitmproxy import options
from mitmproxy import optmanager
from mitmproxy import proxy
from mitmproxy.utils import compat, debug, arg_check
def assert_utf8_env():
spec = ""
for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
spec += os.environ.get(i, "").lower()
if "utf" not in spec:
print(
"Error: mitmproxy requires a UTF console environment.",
file=sys.stderr
)
print(
"Set your LANG environment variable to something like en_US.UTF-8",
file=sys.stderr
)
sys.exit(1)
def process_options(parser, opts, args):
if args.version:
print(debug.dump_system_info())
sys.exit(0)
if args.quiet or args.options or args.commands:
# also reduce log verbosity if --options or --commands is passed,
# we don't want log messages from regular startup then.
args.termlog_verbosity = 'error'
args.flow_detail = 0
if args.verbose:
args.termlog_verbosity = 'debug'
args.flow_detail = 2
adict = {}
for n in dir(args):
if n in opts:
adict[n] = getattr(args, n)
opts.merge(adict)
return proxy.config.ProxyConfig(opts)
def run(
master_cls: typing.Type[master.Master],
make_parser: typing.Callable[[options.Options], argparse.ArgumentParser],
arguments: typing.Sequence[str],
extra: typing.Callable[[typing.Any], dict] = None
) -> master.Master: # pragma: no cover
"""
extra: Extra argument processing callable which returns a dict of
options.
"""
debug.register_info_dumpers()
opts = options.Options()
master = master_cls(opts)
parser = make_parser(opts)
# To make migration from 2.x to 3.0 bearable.
if "-R" in sys.argv and sys.argv[sys.argv.index("-R") + 1].startswith("http"):
print("To use mitmproxy in reverse mode please use --mode reverse:SPEC instead")
try:
args = parser.parse_args(arguments)
except SystemExit:
arg_check.check()
sys.exit(1)
try:
opts.set(*args.setoptions, defer=True)
optmanager.load_paths(
opts,
os.path.join(opts.confdir, "config.yaml"),
os.path.join(opts.confdir, "config.yml"),
)
pconf = process_options(parser, opts, args)
server: typing.Any = None
if pconf.options.server and not compat.new_proxy_core: # new core initializes itself as an addon
try:
server = proxy.server.ProxyServer(pconf)
except exceptions.ServerException as v:
print(str(v), file=sys.stderr)
sys.exit(1)
else:
server = proxy.server.DummyServer(pconf)
master.server = server
if args.options:
print(optmanager.dump_defaults(opts))
sys.exit(0)
if args.commands:
master.commands.dump()
sys.exit(0)
if extra:
if(args.filter_args):
master.log.info(f"Only processing flows that match \"{' & '.join(args.filter_args)}\"")
opts.update(**extra(args))
loop = asyncio.get_event_loop()
try:
loop.add_signal_handler(signal.SIGINT, getattr(master, "prompt_for_exit", master.shutdown))
loop.add_signal_handler(signal.SIGTERM, master.shutdown)
except NotImplementedError:
# Not supported on Windows
pass
# Make sure that we catch KeyboardInterrupts on Windows.
# https://stackoverflow.com/a/36925722/934719
if os.name == "nt":
async def wakeup():
while True:
await asyncio.sleep(0.2)
asyncio.ensure_future(wakeup())
master.run()
except exceptions.OptionsError as e:
print("{}: {}".format(sys.argv[0], e), file=sys.stderr)
sys.exit(1)
except (KeyboardInterrupt, RuntimeError):
pass
return master
def mitmproxy(args=None) -> typing.Optional[int]: # pragma: no cover
if os.name == "nt":
print("Error: mitmproxy's console interface is not supported on Windows. "
"You can run mitmdump or mitmweb instead.", file=sys.stderr)
return 1
assert_utf8_env()
from mitmproxy.tools import console
run(console.master.ConsoleMaster, cmdline.mitmproxy, args)
return None
def mitmdump(args=None) -> typing.Optional[int]: # pragma: no cover
from mitmproxy.tools import dump
def extra(args):
if args.filter_args:
v = " ".join(args.filter_args)
return dict(
save_stream_filter=v,
readfile_filter=v,
dumper_filter=v,
)
return {}
m = run(dump.DumpMaster, cmdline.mitmdump, args, extra)
if m and m.errorcheck.has_errored: # type: ignore
return 1
return None
def mitmweb(args=None) -> typing.Optional[int]: # pragma: no cover
from mitmproxy.tools import web
run(web.master.WebMaster, cmdline.mitmweb, args)
return None

185
mitmproxy/tools/main.py
View File

@ -1,22 +1,169 @@
"""
This file must be kept in a python2.7 and python3.5 compatible syntax!
DO NOT use type annotations or other python3.6-only features that makes this file unparsable by older interpreters!
"""
from __future__ import print_function # this is here for the version check to work on Python 2.
import os
import sys
import asyncio
import argparse
import signal
import typing
if sys.version_info < (3, 6):
# This must be before any mitmproxy imports, as they already break!
# Keep all other imports below with the 'noqa' magic comment.
print("#" * 76, file=sys.stderr)
print("# mitmproxy requires Python 3.6 or higher! #", file=sys.stderr)
print("#" + " " * 74 + "#", file=sys.stderr)
print("# Please upgrade your Python interpreter or use our mitmproxy binaries from #", file=sys.stderr)
print("# https://mitmproxy.org. If your operating system does not include the #", file=sys.stderr)
print("# required Python version, you can try using pyenv or similar tools. #", file=sys.stderr)
print("#" * 76, file=sys.stderr)
from mitmproxy.tools import cmdline
from mitmproxy import exceptions, master
from mitmproxy import options
from mitmproxy import optmanager
from mitmproxy import proxy
from mitmproxy.utils import compat, debug, arg_check
def assert_utf8_env():
spec = ""
for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
spec += os.environ.get(i, "").lower()
if "utf" not in spec:
print(
"Error: mitmproxy requires a UTF console environment.",
file=sys.stderr
)
print(
"Set your LANG environment variable to something like en_US.UTF-8",
file=sys.stderr
)
sys.exit(1)
else:
from ._main import * # noqa
def process_options(parser, opts, args):
if args.version:
print(debug.dump_system_info())
sys.exit(0)
if args.quiet or args.options or args.commands:
# also reduce log verbosity if --options or --commands is passed,
# we don't want log messages from regular startup then.
args.termlog_verbosity = 'error'
args.flow_detail = 0
if args.verbose:
args.termlog_verbosity = 'debug'
args.flow_detail = 2
adict = {}
for n in dir(args):
if n in opts:
adict[n] = getattr(args, n)
opts.merge(adict)
return proxy.config.ProxyConfig(opts)
def run(
master_cls: typing.Type[master.Master],
make_parser: typing.Callable[[options.Options], argparse.ArgumentParser],
arguments: typing.Sequence[str],
extra: typing.Callable[[typing.Any], dict] = None
) -> master.Master: # pragma: no cover
"""
extra: Extra argument processing callable which returns a dict of
options.
"""
debug.register_info_dumpers()
opts = options.Options()
master = master_cls(opts)
parser = make_parser(opts)
# To make migration from 2.x to 3.0 bearable.
if "-R" in sys.argv and sys.argv[sys.argv.index("-R") + 1].startswith("http"):
print("To use mitmproxy in reverse mode please use --mode reverse:SPEC instead")
try:
args = parser.parse_args(arguments)
except SystemExit:
arg_check.check()
sys.exit(1)
try:
opts.set(*args.setoptions, defer=True)
optmanager.load_paths(
opts,
os.path.join(opts.confdir, "config.yaml"),
os.path.join(opts.confdir, "config.yml"),
)
pconf = process_options(parser, opts, args)
server: typing.Any = None
if pconf.options.server and not compat.new_proxy_core: # new core initializes itself as an addon
try:
server = proxy.server.ProxyServer(pconf)
except exceptions.ServerException as v:
print(str(v), file=sys.stderr)
sys.exit(1)
else:
server = proxy.server.DummyServer(pconf)
master.server = server
if args.options:
print(optmanager.dump_defaults(opts))
sys.exit(0)
if args.commands:
master.commands.dump()
sys.exit(0)
if extra:
if(args.filter_args):
master.log.info(f"Only processing flows that match \"{' & '.join(args.filter_args)}\"")
opts.update(**extra(args))
loop = asyncio.get_event_loop()
try:
loop.add_signal_handler(signal.SIGINT, getattr(master, "prompt_for_exit", master.shutdown))
loop.add_signal_handler(signal.SIGTERM, master.shutdown)
except NotImplementedError:
# Not supported on Windows
pass
# Make sure that we catch KeyboardInterrupts on Windows.
# https://stackoverflow.com/a/36925722/934719
if os.name == "nt":
async def wakeup():
while True:
await asyncio.sleep(0.2)
asyncio.ensure_future(wakeup())
master.run()
except exceptions.OptionsError as e:
print("{}: {}".format(sys.argv[0], e), file=sys.stderr)
sys.exit(1)
except (KeyboardInterrupt, RuntimeError):
pass
return master
def mitmproxy(args=None) -> typing.Optional[int]: # pragma: no cover
if os.name == "nt":
print("Error: mitmproxy's console interface is not supported on Windows. "
"You can run mitmdump or mitmweb instead.", file=sys.stderr)
return 1
assert_utf8_env()
from mitmproxy.tools import console
run(console.master.ConsoleMaster, cmdline.mitmproxy, args)
return None
def mitmdump(args=None) -> typing.Optional[int]: # pragma: no cover
from mitmproxy.tools import dump
def extra(args):
if args.filter_args:
v = " ".join(args.filter_args)
return dict(
save_stream_filter=v,
readfile_filter=v,
dumper_filter=v,
)
return {}
m = run(dump.DumpMaster, cmdline.mitmdump, args, extra)
if m and m.errorcheck.has_errored: # type: ignore
return 1
return None
def mitmweb(args=None) -> typing.Optional[int]: # pragma: no cover
from mitmproxy.tools import web
run(web.master.WebMaster, cmdline.mitmweb, args)
return None

2
mitmproxy/utils/debug.py
View File

@ -83,8 +83,6 @@ def dump_info(signal=None, frame=None, file=sys.stdout, testing=False): # pragm
print(i[1], i[0])
try:
if sys.version_info < (3, 8):
raise RuntimeError
asyncio.get_running_loop()
except RuntimeError:
pass

10
setup.py
View File

@ -34,8 +34,6 @@ setup(
"Operating System :: POSIX",
"Operating System :: Microsoft :: Windows",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: Implementation :: CPython",
@ -65,6 +63,7 @@ setup(
"pathoc = pathod.pathoc_cmdline:go_pathoc"
]
},
python_requires='>=3.8',
# https://packaging.python.org/en/latest/requirements/#install-requires
# It is not considered best practice to use install_requires to pin dependencies to specific versions.
install_requires=[
@ -75,8 +74,8 @@ setup(
"click>=7.0,<8",
"cryptography>=3.2,<3.3",
"flask>=1.1.1,<1.2",
"h2>=4.0,<5; python_version>='3.6.0'", # python_version only needed to make "py36+ required" message work
"hyperframe>=6.0,<7; python_version>='3.6.0'", # python_version only needed to make "py36+ required" message work
"h2>=4.0,<5",
"hyperframe>=6.0,<7",
"kaitaistruct>=0.7,<0.10",
"ldap3>=2.8,<2.9",
"msgpack>=1.0.0, <1.1.0",
@ -98,9 +97,6 @@ setup(
':sys_platform == "win32"': [
"pydivert>=2.0.3,<2.2",
],
':python_version == "3.6"': [
"dataclasses>=0.7",
],
'dev': [
"hypothesis>=5.8,<6",
"parver>=0.1,<2.0",

4
test/mitmproxy/addons/test_readfile.py
View File

@ -1,6 +1,5 @@
import asyncio
import io
import sys
import pytest
from unittest import mock
@ -47,7 +46,6 @@ class TestReadFile:
with pytest.raises(Exception, match="Invalid readfile filter"):
tctx.configure(rf, readfile_filter="~~")
@pytest.mark.skipif(sys.version_info < (3, 8), reason='requires Python 3.8 or higher')
@pytest.mark.asyncio
async def test_read(self, tmpdir, data, corrupt_data):
rf = readfile.ReadFile()
@ -95,7 +93,6 @@ class TestReadFile:
class TestReadFileStdin:
@pytest.mark.skipif(sys.version_info < (3, 8), reason='requires Python 3.8 or higher')
@mock.patch('sys.stdin')
@pytest.mark.asyncio
async def test_stdin(self, stdin, data, corrupt_data):
@ -111,7 +108,6 @@ class TestReadFileStdin:
with pytest.raises(exceptions.FlowReadException):
await rf.load_flows(stdin.buffer)
@pytest.mark.skipif(sys.version_info < (3, 8), reason='requires Python 3.8 or higher')
@pytest.mark.asyncio
async def test_normal(self, tmpdir, data):
rf = readfile.ReadFileStdin()

2
test/mitmproxy/net/http/test_url.py
View File

@ -1,7 +1,6 @@
from typing import AnyStr
import pytest
import sys
from mitmproxy.net.http import url
from mitmproxy.net.http.url import parse_authority
@ -62,7 +61,6 @@ def test_ascii_check():
b'%BD%E7%8C%AB%E6%B0%93%E7%8C%AB%E6%B0%93'
@pytest.mark.skipif(sys.version_info < (3, 6), reason='requires Python 3.6 or higher')
def test_parse_port_range():
# Port out of range
with pytest.raises(ValueError):

7
test/mitmproxy/utils/test_asyncio_utils.py
View File

@ -1,5 +1,4 @@
import asyncio
import sys
import pytest
@ -15,7 +14,6 @@ async def ttask():
@pytest.mark.asyncio
@pytest.mark.skipif(sys.version_info < (3, 8), reason="requires Python 3.8")
async def test_simple():
task = asyncio_utils.create_task(
ttask(),
@ -30,7 +28,6 @@ async def test_simple():
assert task.cancelled()
@pytest.mark.skipif(sys.version_info < (3, 8), reason="requires Python 3.8")
def test_closed_loop():
# Crude test for line coverage.
# This should eventually go, see the description in asyncio_utils.create_task for details.
@ -38,9 +35,11 @@ def test_closed_loop():
ttask(),
name="ttask",
)
t = ttask()
with pytest.raises(RuntimeError):
asyncio_utils.create_task(
ttask(),
t,
name="ttask",
ignore_closed_loop=False,
)
t.close() # suppress "not awaited" warning

1
test/mitmproxy/utils/test_debug.py
View File

@ -24,7 +24,6 @@ def test_dump_info():
async def test_dump_info_async():
cs = io.StringIO()
debug.dump_info(None, None, file=cs, testing=True)
if sys.version_info >= (3, 8):
assert "Tasks" in cs.getvalue()

12
tox.ini
View File

@ -1,5 +1,5 @@
[tox]
envlist = py35, py36, py37, py38, py39, flake8, filename_matching, mypy, individual_coverage, docs
envlist = py38, py39, flake8, filename_matching, mypy, individual_coverage, docs
skipsdist = True
toxworkdir={env:TOX_WORK_DIR:.tox}
@ -14,14 +14,6 @@ commands =
--full-cov=mitmproxy/ --full-cov=pathod/ \
{posargs}
[testenv:py35]
whitelist_externals =
bash
deps =
-rrequirements.txt
commands =
bash -c "mitmdump --version 2>&1 | grep 'mitmproxy requires Python 3.6'"
[testenv:flake8]
deps = flake8==3.8.4
commands =
@ -51,7 +43,7 @@ commands =
passenv = CI_* GITHUB_* AWS_* TWINE_* DOCKER_*
deps =
-rrequirements.txt
pyinstaller==4.0
pyinstaller==4.1
twine==3.2.0
awscli
commands =