mirror of
https://github.com/Grasscutters/mitmproxy.git
synced 2024-11-22 07:08:10 +00:00
400 lines
12 KiB
Python
Executable File
400 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import glob
|
|
import re
|
|
import contextlib
|
|
import os
|
|
import platform
|
|
import sys
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
import zipfile
|
|
from os.path import join, abspath, dirname, exists, basename
|
|
|
|
import click
|
|
import cryptography.fernet
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def chdir(path: str):
|
|
old_dir = os.getcwd()
|
|
os.chdir(path)
|
|
yield
|
|
os.chdir(old_dir)
|
|
|
|
|
|
class BuildError(Exception):
|
|
pass
|
|
|
|
|
|
class BuildEnviron:
|
|
PLATFORM_TAGS = {
|
|
"Darwin": "osx",
|
|
"Windows": "windows",
|
|
"Linux": "linux",
|
|
}
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
system = "",
|
|
root_dir = "",
|
|
|
|
travis_tag = "",
|
|
travis_branch = "",
|
|
travis_pull_request = "",
|
|
|
|
appveyor_repo_tag_name = "",
|
|
appveyor_repo_branch = "",
|
|
appveyor_pull_request_number = "",
|
|
|
|
build_wheel = "",
|
|
build_docker = "",
|
|
build_pyinstaller = "",
|
|
|
|
has_aws_creds = False,
|
|
has_twine_creds = False,
|
|
|
|
docker_username = "",
|
|
docker_password = "",
|
|
):
|
|
self.system = system
|
|
self.root_dir = root_dir
|
|
|
|
self.travis_tag = travis_tag
|
|
self.travis_branch = travis_branch
|
|
self.travis_pull_request = travis_pull_request
|
|
|
|
self.appveyor_repo_tag_name = appveyor_repo_tag_name
|
|
self.appveyor_repo_branch = appveyor_repo_branch
|
|
self.appveyor_pull_request_number = appveyor_pull_request_number
|
|
|
|
self.has_aws_creds = has_aws_creds
|
|
self.has_twine_creds = has_twine_creds
|
|
self.docker_username = docker_username
|
|
self.docker_password = docker_password
|
|
|
|
@classmethod
|
|
def from_env(klass):
|
|
return klass(
|
|
system = platform.system,
|
|
root_dir = dirname(__file__),
|
|
|
|
travis_tag = os.environ.get("TRAVIS_TAG", ""),
|
|
travis_branch = os.environ.get("TRAVIS_BRANCH", ""),
|
|
travis_pull_request = os.environ.get("TRAVIS_PULL_REQUEST"),
|
|
|
|
appveyor_repo_tag_name = os.environ.get("APPVEYOR_REPO_TAG_NAME", ""),
|
|
appveyor_repo_branch = os.environ.get("APPVEYOR_REPO_BRANCH", ""),
|
|
appveyor_pull_request_number = os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"),
|
|
|
|
build_wheel = "WHEEL" in os.environ,
|
|
build_pyinstaller = "PYINSTALLER" in os.environ,
|
|
build_docker = "DOCKER" in os.environ,
|
|
|
|
has_aws_creds = "AWS_ACCESS_KEY_ID" in os.environ,
|
|
has_twine_creds= (
|
|
"TWINE_USERNAME" in os.environ and
|
|
"TWINE_PASSWORD" in os.environ
|
|
),
|
|
|
|
docker_username = os.environ.get("DOCKER_USERNAME"),
|
|
docker_password = os.environ.get("DOCKER_PASSWORD"),
|
|
)
|
|
|
|
@property
|
|
def has_docker_creds(self) -> bool:
|
|
return self.docker_username and self.docker_password
|
|
|
|
@property
|
|
def is_pull_request(self) -> bool:
|
|
if self.appveyor_pull_request_number:
|
|
return True
|
|
if self.travis_pull_request and self.travis_pull_request != "false":
|
|
return True
|
|
return False
|
|
|
|
@property
|
|
def tag(self):
|
|
return self.travis_tag or self.appveyor_repo_tag_name
|
|
|
|
@property
|
|
def branch(self):
|
|
return self.travis_branch or self.appveyor_repo_branch
|
|
|
|
@property
|
|
def version(self):
|
|
name = self.tag or self.branch
|
|
if not name:
|
|
raise BuildError("Could not establish build name")
|
|
return re.sub('^v', "", name)
|
|
|
|
@property
|
|
def upload_dir(self):
|
|
if self.tag:
|
|
return self.version
|
|
else:
|
|
return "branches/%s" % self.version
|
|
|
|
@property
|
|
def platform_tag(self):
|
|
if self.system in self.PLATFORM_TAGS:
|
|
return self.PLATFORM_TAGS[self.system]
|
|
raise BuildError("Unsupported platform: %s" % self.system)
|
|
|
|
@property
|
|
def release_dir(self):
|
|
return os.path.join(self.root_dir, "release")
|
|
|
|
@property
|
|
def build_dir(self):
|
|
return os.path.join(self.release_dir, "build")
|
|
|
|
@property
|
|
def dist_dir(self):
|
|
return os.path.join(self.release_dir, "dist")
|
|
|
|
def archive(self, name):
|
|
# ZipFile and tarfile have slightly different APIs. Fix that.
|
|
if self.system == "Windows":
|
|
a = zipfile.ZipFile(name, "w")
|
|
a.add = a.write
|
|
return a
|
|
else:
|
|
return tarfile.open(name, "w:gz")
|
|
|
|
def archive_name(self, bdist: str) -> str:
|
|
if self.system == "Windows":
|
|
ext = "zip"
|
|
else:
|
|
ext = "tar.gz"
|
|
return "{project}-{version}-{platform}.{ext}".format(
|
|
project=bdist,
|
|
version=self.version,
|
|
platform=self.platform_tag,
|
|
ext=ext
|
|
)
|
|
|
|
@property
|
|
def bdists(self):
|
|
ret = {
|
|
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
|
|
"pathod": ["pathoc", "pathod"]
|
|
}
|
|
if self.system == "Windows":
|
|
ret["mitmproxy"].remove("mitmproxy")
|
|
return ret
|
|
|
|
def dump_info(self, fp=sys.stdout):
|
|
print("BUILD PLATFORM_TAG=%s" % self.platform_tag, file=fp)
|
|
print("BUILD ROOT_DIR=%s" % self.root_dir, file=fp)
|
|
print("BUILD RELEASE_DIR=%s" % self.release_dir, file=fp)
|
|
print("BUILD BUILD_DIR=%s" % self.build_dir, file=fp)
|
|
print("BUILD DIST_DIR=%s" % self.dist_dir, file=fp)
|
|
print("BUILD BDISTS=%s" % self.bdists, file=fp)
|
|
print("BUILD TAG=%s" % self.tag, file=fp)
|
|
print("BUILD BRANCH=%s" % self.branch, file=fp)
|
|
print("BUILD VERSION=%s" % self.version, file=fp)
|
|
print("BUILD UPLOAD_DIR=%s" % self.upload_dir, file=fp)
|
|
|
|
|
|
def build_wheel(be: BuildEnviron):
|
|
click.echo("Building wheel...")
|
|
subprocess.check_call([
|
|
"python",
|
|
"setup.py",
|
|
"-q",
|
|
"bdist_wheel",
|
|
"--dist-dir", be.dist_dir,
|
|
])
|
|
whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
|
|
click.echo("Found wheel package: {}".format(whl))
|
|
subprocess.check_call(["tox", "-e", "wheeltest", "--", whl])
|
|
return whl
|
|
|
|
|
|
def build_docker_image(be: BuildEnviron, whl: str):
|
|
click.echo("Building Docker image...")
|
|
subprocess.check_call([
|
|
"docker",
|
|
"build",
|
|
"--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, be.root_dir)),
|
|
"--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(basename(whl)),
|
|
"--file", "docker/Dockerfile",
|
|
"."
|
|
])
|
|
|
|
|
|
def build_pyinstaller(be: BuildEnviron):
|
|
click.echo("Building pyinstaller package...")
|
|
|
|
PYINSTALLER_SPEC = join(be.release_dir, "specs")
|
|
# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
|
|
PYINSTALLER_HOOKS = join(be.release_dir, "hooks")
|
|
PYINSTALLER_TEMP = join(be.build_dir, "pyinstaller")
|
|
PYINSTALLER_DIST = join(be.build_dir, "binaries", be.platform_tag)
|
|
|
|
# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
|
|
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
|
|
if platform.system() == "Windows":
|
|
PYINSTALLER_ARGS = [
|
|
# PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
|
|
"-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
|
|
]
|
|
else:
|
|
PYINSTALLER_ARGS = []
|
|
|
|
if exists(PYINSTALLER_TEMP):
|
|
shutil.rmtree(PYINSTALLER_TEMP)
|
|
if exists(PYINSTALLER_DIST):
|
|
shutil.rmtree(PYINSTALLER_DIST)
|
|
|
|
for bdist, tools in sorted(be.bdists.items()):
|
|
with be.archive(join(be.dist_dir, be.archive_name(bdist))) as archive:
|
|
for tool in tools:
|
|
# We can't have a folder and a file with the same name.
|
|
if tool == "mitmproxy":
|
|
tool = "mitmproxy_main"
|
|
# This is PyInstaller, so it messes up paths.
|
|
# We need to make sure that we are in the spec folder.
|
|
with chdir(PYINSTALLER_SPEC):
|
|
click.echo("Building PyInstaller %s binary..." % tool)
|
|
excludes = []
|
|
if tool != "mitmweb":
|
|
excludes.append("mitmproxy.tools.web")
|
|
if tool != "mitmproxy_main":
|
|
excludes.append("mitmproxy.tools.console")
|
|
|
|
subprocess.check_call(
|
|
[
|
|
"pyinstaller",
|
|
"--clean",
|
|
"--workpath", PYINSTALLER_TEMP,
|
|
"--distpath", PYINSTALLER_DIST,
|
|
"--additional-hooks-dir", PYINSTALLER_HOOKS,
|
|
"--onefile",
|
|
"--console",
|
|
"--icon", "icon.ico",
|
|
# This is PyInstaller, so setting a
|
|
# different log level obviously breaks it :-)
|
|
# "--log-level", "WARN",
|
|
]
|
|
+ [x for e in excludes for x in ["--exclude-module", e]]
|
|
+ PYINSTALLER_ARGS
|
|
+ [tool]
|
|
)
|
|
# Delete the spec file - we're good without.
|
|
os.remove("{}.spec".format(tool))
|
|
|
|
# Test if it works at all O:-)
|
|
executable = join(PYINSTALLER_DIST, tool)
|
|
if platform.system() == "Windows":
|
|
executable += ".exe"
|
|
|
|
# Remove _main suffix from mitmproxy executable
|
|
if "_main" in executable:
|
|
shutil.move(
|
|
executable,
|
|
executable.replace("_main", "")
|
|
)
|
|
executable = executable.replace("_main", "")
|
|
|
|
click.echo("> %s --version" % executable)
|
|
click.echo(subprocess.check_output([executable, "--version"]).decode())
|
|
|
|
archive.add(executable, basename(executable))
|
|
click.echo("Packed {}.".format(be.archive_name(bdist)))
|
|
|
|
|
|
@click.group(chain=True)
|
|
def cli():
|
|
"""
|
|
mitmproxy build tool
|
|
"""
|
|
pass
|
|
|
|
|
|
@cli.command("build")
|
|
def build():
|
|
"""
|
|
Build a binary distribution
|
|
"""
|
|
be = BuildEnviron.from_env()
|
|
be.dump_info()
|
|
|
|
os.makedirs(be.dist_dir, exist_ok=True)
|
|
|
|
if be.build_wheel:
|
|
whl = build_wheel(be)
|
|
# Docker image requires wheels
|
|
if be.build_docker:
|
|
build_docker_image(whl)
|
|
if be.build_pyinstaller:
|
|
build_pyinstaller()
|
|
|
|
|
|
@cli.command("upload")
|
|
def upload():
|
|
"""
|
|
Upload build artifacts
|
|
|
|
Uploads the wheels package to PyPi.
|
|
Uploads the Pyinstaller and wheels packages to the snapshot server.
|
|
Pushes the Docker image to Docker Hub.
|
|
"""
|
|
be = BuildEnviron.from_env()
|
|
|
|
if be.is_pull_request:
|
|
click.echo("Refusing to upload artifacts from a pull request!")
|
|
return
|
|
|
|
if be.has_aws_creds:
|
|
subprocess.check_call([
|
|
"aws", "s3", "cp",
|
|
"--acl", "public-read",
|
|
be.dist_dir + "/",
|
|
"s3://snapshots.mitmproxy.org/{}/".format(be.upload_dir),
|
|
"--recursive",
|
|
])
|
|
|
|
upload_pypi = (be.tag and be.build_wheel and be.has_twine_creds)
|
|
if upload_pypi:
|
|
whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
|
|
click.echo("Uploading {} to PyPi...".format(whl))
|
|
subprocess.check_call(["twine", "upload", whl])
|
|
|
|
upload_docker = (
|
|
(be.tag or be.branch == "master") and
|
|
be.build_docker,
|
|
be.has_docker_creds,
|
|
)
|
|
if upload_docker:
|
|
docker_tag = "dev" if be.branch == "master" else be.version
|
|
|
|
click.echo("Uploading Docker image to tag={}...".format(docker_tag))
|
|
subprocess.check_call([
|
|
"docker",
|
|
"login",
|
|
"-u", be.docker_username,
|
|
"-p", be.docker_password,
|
|
])
|
|
subprocess.check_call([
|
|
"docker",
|
|
"push",
|
|
"mitmproxy/mitmproxy:{}".format(docker_tag),
|
|
])
|
|
|
|
|
|
@cli.command("decrypt")
|
|
@click.argument('infile', type=click.File('rb'))
|
|
@click.argument('outfile', type=click.File('wb'))
|
|
@click.argument('key', envvar='RTOOL_KEY')
|
|
def decrypt(infile, outfile, key):
|
|
f = cryptography.fernet.Fernet(key.encode())
|
|
outfile.write(f.decrypt(infile.read()))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli()
|