mirror of
https://github.com/Grasscutters/mitmproxy.git
synced 2024-11-21 22:58:24 +00:00
cibuild: Consolidate build environment and add tests
This commit is contained in:
Aldo Cortesi
parent
f910f361a0
commit
8fceaca6b8
0
release/__init__.py
Normal file
0
release/__init__.py
Normal file
@ -15,76 +15,6 @@ from os.path import join, abspath, dirname, exists, basename
|
||||
import click
|
||||
import cryptography.fernet
|
||||
|
||||
# ZipFile and tarfile have slightly different APIs. Fix that.
|
||||
if platform.system() == "Windows":
|
||||
def Archive(name):
|
||||
a = zipfile.ZipFile(name, "w")
|
||||
a.add = a.write
|
||||
return a
|
||||
else:
|
||||
def Archive(name):
|
||||
return tarfile.open(name, "w:gz")
|
||||
|
||||
PLATFORM_TAG = {
|
||||
"Darwin": "osx",
|
||||
"Windows": "windows",
|
||||
"Linux": "linux",
|
||||
}.get(platform.system(), platform.system())
|
||||
|
||||
ROOT_DIR = abspath(join(dirname(__file__), ".."))
|
||||
RELEASE_DIR = join(ROOT_DIR, "release")
|
||||
BUILD_DIR = join(RELEASE_DIR, "build")
|
||||
DIST_DIR = join(RELEASE_DIR, "dist")
|
||||
|
||||
BDISTS = {
|
||||
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
|
||||
"pathod": ["pathoc", "pathod"]
|
||||
}
|
||||
if platform.system() == "Windows":
|
||||
BDISTS["mitmproxy"].remove("mitmproxy")
|
||||
|
||||
TOOLS = [
|
||||
tool
|
||||
for tools in sorted(BDISTS.values())
|
||||
for tool in tools
|
||||
]
|
||||
|
||||
TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
|
||||
BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
|
||||
if TAG:
|
||||
VERSION = re.sub('^v', '', TAG)
|
||||
UPLOAD_DIR = VERSION
|
||||
elif BRANCH:
|
||||
VERSION = re.sub('^v', '', BRANCH)
|
||||
UPLOAD_DIR = "branches/%s" % VERSION
|
||||
else:
|
||||
print("Could not establish build name - exiting." % BRANCH)
|
||||
sys.exit(0)
|
||||
|
||||
print("BUILD PLATFORM_TAG=%s" % PLATFORM_TAG)
|
||||
print("BUILD ROOT_DIR=%s" % ROOT_DIR)
|
||||
print("BUILD RELEASE_DIR=%s" % RELEASE_DIR)
|
||||
print("BUILD BUILD_DIR=%s" % BUILD_DIR)
|
||||
print("BUILD DIST_DIR=%s" % DIST_DIR)
|
||||
print("BUILD BDISTS=%s" % BDISTS)
|
||||
print("BUILD TAG=%s" % TAG)
|
||||
print("BUILD BRANCH=%s" % BRANCH)
|
||||
print("BUILD VERSION=%s" % VERSION)
|
||||
print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR)
|
||||
|
||||
|
||||
def archive_name(bdist: str) -> str:
|
||||
if platform.system() == "Windows":
|
||||
ext = "zip"
|
||||
else:
|
||||
ext = "tar.gz"
|
||||
return "{project}-{version}-{platform}.{ext}".format(
|
||||
project=bdist,
|
||||
version=VERSION,
|
||||
platform=PLATFORM_TAG,
|
||||
ext=ext
|
||||
)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def chdir(path: str):
|
||||
@ -94,79 +24,216 @@ def chdir(path: str):
|
||||
os.chdir(old_dir)
|
||||
|
||||
|
||||
@click.group(chain=True)
|
||||
def cli():
|
||||
"""
|
||||
mitmproxy build tool
|
||||
"""
|
||||
class BuildError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@cli.command("build")
|
||||
def build():
|
||||
"""
|
||||
Build a binary distribution
|
||||
"""
|
||||
os.makedirs(DIST_DIR, exist_ok=True)
|
||||
class BuildEnviron:
|
||||
PLATFORM_TAGS = {
|
||||
"Darwin": "osx",
|
||||
"Windows": "windows",
|
||||
"Linux": "linux",
|
||||
}
|
||||
|
||||
if "WHEEL" in os.environ:
|
||||
whl = build_wheel()
|
||||
else:
|
||||
click.echo("Not building wheels.")
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
system = "",
|
||||
root_dir = "",
|
||||
|
||||
if "WHEEL" in os.environ and "DOCKER" in os.environ:
|
||||
# Docker image requires wheels
|
||||
build_docker_image(whl)
|
||||
else:
|
||||
click.echo("Not building Docker image.")
|
||||
travis_tag = "",
|
||||
travis_branch = "",
|
||||
travis_pull_request = "",
|
||||
|
||||
if "PYINSTALLER" in os.environ:
|
||||
build_pyinstaller()
|
||||
else:
|
||||
click.echo("Not building PyInstaller packages.")
|
||||
appveyor_repo_tag_name = "",
|
||||
appveyor_repo_branch = "",
|
||||
appveyor_pull_request_number = "",
|
||||
|
||||
build_wheel = "",
|
||||
build_docker = "",
|
||||
build_pyinstaller = "",
|
||||
|
||||
has_aws_creds = False,
|
||||
has_twine_creds = False,
|
||||
|
||||
docker_username = "",
|
||||
docker_password = "",
|
||||
):
|
||||
self.system = system
|
||||
self.root_dir = root_dir
|
||||
|
||||
self.travis_tag = travis_tag
|
||||
self.travis_branch = travis_branch
|
||||
self.travis_pull_request = travis_pull_request
|
||||
|
||||
self.appveyor_repo_tag_name = appveyor_repo_tag_name
|
||||
self.appveyor_repo_branch = appveyor_repo_branch
|
||||
self.appveyor_pull_request_number = appveyor_pull_request_number
|
||||
|
||||
self.has_aws_creds = has_aws_creds
|
||||
self.has_twine_creds = has_twine_creds
|
||||
self.docker_username = docker_username
|
||||
self.docker_password = docker_password
|
||||
|
||||
@classmethod
|
||||
def from_env(klass):
|
||||
return klass(
|
||||
system = platform.system,
|
||||
root_dir = dirname(__file__),
|
||||
|
||||
travis_tag = os.environ.get("TRAVIS_TAG", ""),
|
||||
travis_branch = os.environ.get("TRAVIS_BRANCH", ""),
|
||||
travis_pull_request = os.environ.get("TRAVIS_PULL_REQUEST"),
|
||||
|
||||
appveyor_repo_tag_name = os.environ.get("APPVEYOR_REPO_TAG_NAME", ""),
|
||||
appveyor_repo_branch = os.environ.get("APPVEYOR_REPO_BRANCH", ""),
|
||||
appveyor_pull_request_number = os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"),
|
||||
|
||||
build_wheel = "WHEEL" in os.environ,
|
||||
build_pyinstaller = "PYINSTALLER" in os.environ,
|
||||
build_docker = "DOCKER" in os.environ,
|
||||
|
||||
has_aws_creds = "AWS_ACCESS_KEY_ID" in os.environ,
|
||||
has_twine_creds= (
|
||||
"TWINE_USERNAME" in os.environ and
|
||||
"TWINE_PASSWORD" in os.environ
|
||||
),
|
||||
|
||||
docker_username = os.environ.get("DOCKER_USERNAME"),
|
||||
docker_password = os.environ.get("DOCKER_PASSWORD"),
|
||||
)
|
||||
|
||||
@property
|
||||
def has_docker_creds(self) -> bool:
|
||||
return self.docker_username and self.docker_password
|
||||
|
||||
@property
|
||||
def is_pull_request(self) -> bool:
|
||||
if self.appveyor_pull_request_number:
|
||||
return True
|
||||
if self.travis_pull_request and self.travis_pull_request != "false":
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def tag(self):
|
||||
return self.travis_tag or self.appveyor_repo_tag_name
|
||||
|
||||
@property
|
||||
def branch(self):
|
||||
return self.travis_branch or self.appveyor_repo_branch
|
||||
|
||||
@property
|
||||
def version(self):
|
||||
name = self.tag or self.branch
|
||||
if not name:
|
||||
raise BuildError("Could not establish build name")
|
||||
return re.sub('^v', "", name)
|
||||
|
||||
@property
|
||||
def upload_dir(self):
|
||||
if self.tag:
|
||||
return self.version
|
||||
else:
|
||||
return "branches/%s" % self.version
|
||||
|
||||
@property
|
||||
def platform_tag(self):
|
||||
if self.system in self.PLATFORM_TAGS:
|
||||
return self.PLATFORM_TAGS[self.system]
|
||||
raise BuildError("Unsupported platform: %s" % self.system)
|
||||
|
||||
@property
|
||||
def release_dir(self):
|
||||
return os.path.join(self.root_dir, "release")
|
||||
|
||||
@property
|
||||
def build_dir(self):
|
||||
return os.path.join(self.release_dir, "build")
|
||||
|
||||
@property
|
||||
def dist_dir(self):
|
||||
return os.path.join(self.release_dir, "dist")
|
||||
|
||||
def archive(self, name):
|
||||
# ZipFile and tarfile have slightly different APIs. Fix that.
|
||||
if self.system == "Windows":
|
||||
a = zipfile.ZipFile(name, "w")
|
||||
a.add = a.write
|
||||
return a
|
||||
else:
|
||||
return tarfile.open(name, "w:gz")
|
||||
|
||||
def archive_name(self, bdist: str) -> str:
|
||||
if self.system == "Windows":
|
||||
ext = "zip"
|
||||
else:
|
||||
ext = "tar.gz"
|
||||
return "{project}-{version}-{platform}.{ext}".format(
|
||||
project=bdist,
|
||||
version=self.version,
|
||||
platform=self.platform_tag,
|
||||
ext=ext
|
||||
)
|
||||
|
||||
@property
|
||||
def bdists(self):
|
||||
ret = {
|
||||
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
|
||||
"pathod": ["pathoc", "pathod"]
|
||||
}
|
||||
if self.system == "Windows":
|
||||
ret["mitmproxy"].remove("mitmproxy")
|
||||
return ret
|
||||
|
||||
def dump_info(self, fp=sys.stdout):
|
||||
print("BUILD PLATFORM_TAG=%s" % self.platform_tag, file=fp)
|
||||
print("BUILD ROOT_DIR=%s" % self.root_dir, file=fp)
|
||||
print("BUILD RELEASE_DIR=%s" % self.release_dir, file=fp)
|
||||
print("BUILD BUILD_DIR=%s" % self.build_dir, file=fp)
|
||||
print("BUILD DIST_DIR=%s" % self.dist_dir, file=fp)
|
||||
print("BUILD BDISTS=%s" % self.bdists, file=fp)
|
||||
print("BUILD TAG=%s" % self.tag, file=fp)
|
||||
print("BUILD BRANCH=%s" % self.branch, file=fp)
|
||||
print("BUILD VERSION=%s" % self.version, file=fp)
|
||||
print("BUILD UPLOAD_DIR=%s" % self.upload_dir, file=fp)
|
||||
|
||||
|
||||
def build_wheel():
|
||||
def build_wheel(be: BuildEnviron):
|
||||
click.echo("Building wheel...")
|
||||
subprocess.check_call([
|
||||
"python",
|
||||
"setup.py",
|
||||
"-q",
|
||||
"bdist_wheel",
|
||||
"--dist-dir", DIST_DIR,
|
||||
"--dist-dir", be.dist_dir,
|
||||
])
|
||||
|
||||
whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
|
||||
whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
|
||||
click.echo("Found wheel package: {}".format(whl))
|
||||
|
||||
subprocess.check_call([
|
||||
"tox",
|
||||
"-e", "wheeltest",
|
||||
"--",
|
||||
whl
|
||||
])
|
||||
|
||||
subprocess.check_call(["tox", "-e", "wheeltest", "--", whl])
|
||||
return whl
|
||||
|
||||
|
||||
def build_docker_image(whl):
|
||||
def build_docker_image(be: BuildEnviron, whl: str):
|
||||
click.echo("Building Docker image...")
|
||||
subprocess.check_call([
|
||||
"docker",
|
||||
"build",
|
||||
"--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, ROOT_DIR)),
|
||||
"--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, be.root_dir)),
|
||||
"--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(basename(whl)),
|
||||
"--file", "docker/Dockerfile",
|
||||
"."
|
||||
])
|
||||
|
||||
|
||||
def build_pyinstaller():
|
||||
PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
|
||||
def build_pyinstaller(be: BuildEnviron):
|
||||
click.echo("Building pyinstaller package...")
|
||||
|
||||
PYINSTALLER_SPEC = join(be.release_dir, "specs")
|
||||
# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
|
||||
PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
|
||||
PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
|
||||
PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
|
||||
PYINSTALLER_HOOKS = join(be.release_dir, "hooks")
|
||||
PYINSTALLER_TEMP = join(be.build_dir, "pyinstaller")
|
||||
PYINSTALLER_DIST = join(be.build_dir, "binaries", be.platform_tag)
|
||||
|
||||
# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
|
||||
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
|
||||
@ -183,8 +250,8 @@ def build_pyinstaller():
|
||||
if exists(PYINSTALLER_DIST):
|
||||
shutil.rmtree(PYINSTALLER_DIST)
|
||||
|
||||
for bdist, tools in sorted(BDISTS.items()):
|
||||
with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
|
||||
for bdist, tools in sorted(be.bdists.items()):
|
||||
with be.archive(join(be.dist_dir, be.archive_name(bdist))) as archive:
|
||||
for tool in tools:
|
||||
# We can't have a folder and a file with the same name.
|
||||
if tool == "mitmproxy":
|
||||
@ -237,7 +304,34 @@ def build_pyinstaller():
|
||||
click.echo(subprocess.check_output([executable, "--version"]).decode())
|
||||
|
||||
archive.add(executable, basename(executable))
|
||||
click.echo("Packed {}.".format(archive_name(bdist)))
|
||||
click.echo("Packed {}.".format(be.archive_name(bdist)))
|
||||
|
||||
|
||||
@click.group(chain=True)
|
||||
def cli():
|
||||
"""
|
||||
mitmproxy build tool
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command("build")
|
||||
def build():
|
||||
"""
|
||||
Build a binary distribution
|
||||
"""
|
||||
be = BuildEnviron.from_env()
|
||||
be.dump_info()
|
||||
|
||||
os.makedirs(be.dist_dir, exist_ok=True)
|
||||
|
||||
if be.build_wheel:
|
||||
whl = build_wheel(be)
|
||||
# Docker image requires wheels
|
||||
if be.build_docker:
|
||||
build_docker_image(whl)
|
||||
if be.build_pyinstaller:
|
||||
build_pyinstaller()
|
||||
|
||||
|
||||
@cli.command("upload")
|
||||
@ -249,57 +343,41 @@ def upload():
|
||||
Uploads the Pyinstaller and wheels packages to the snapshot server.
|
||||
Pushes the Docker image to Docker Hub.
|
||||
"""
|
||||
be = BuildEnviron.from_env()
|
||||
|
||||
# Our credentials are only available from within the main repository and not forks.
|
||||
# We need to prevent uploads from all BUT the branches in the main repository.
|
||||
# Pull requests and master-branches of forks are not allowed to upload.
|
||||
is_pull_request = (
|
||||
("TRAVIS_PULL_REQUEST" in os.environ and os.environ["TRAVIS_PULL_REQUEST"] != "false") or
|
||||
"APPVEYOR_PULL_REQUEST_NUMBER" in os.environ
|
||||
)
|
||||
if is_pull_request:
|
||||
if be.is_pull_request:
|
||||
click.echo("Refusing to upload artifacts from a pull request!")
|
||||
return
|
||||
|
||||
if "AWS_ACCESS_KEY_ID" in os.environ:
|
||||
if be.has_aws_creds:
|
||||
subprocess.check_call([
|
||||
"aws", "s3", "cp",
|
||||
"--acl", "public-read",
|
||||
DIST_DIR + "/",
|
||||
"s3://snapshots.mitmproxy.org/{}/".format(UPLOAD_DIR),
|
||||
be.dist_dir + "/",
|
||||
"s3://snapshots.mitmproxy.org/{}/".format(be.upload_dir),
|
||||
"--recursive",
|
||||
])
|
||||
|
||||
upload_pypi = (
|
||||
TAG and
|
||||
"WHEEL" in os.environ and
|
||||
"TWINE_USERNAME" in os.environ and
|
||||
"TWINE_PASSWORD" in os.environ
|
||||
)
|
||||
upload_pypi = (be.tag and be.build_wheel and be.has_twine_creds)
|
||||
if upload_pypi:
|
||||
whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
|
||||
whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
|
||||
click.echo("Uploading {} to PyPi...".format(whl))
|
||||
subprocess.check_call([
|
||||
"twine",
|
||||
"upload",
|
||||
whl
|
||||
])
|
||||
subprocess.check_call(["twine", "upload", whl])
|
||||
|
||||
upload_docker = (
|
||||
(TAG or BRANCH == "master") and
|
||||
"DOCKER" in os.environ and
|
||||
"DOCKER_USERNAME" in os.environ and
|
||||
"DOCKER_PASSWORD" in os.environ
|
||||
(be.tag or be.branch == "master") and
|
||||
be.build_docker,
|
||||
be.has_docker_creds,
|
||||
)
|
||||
if upload_docker:
|
||||
docker_tag = "dev" if BRANCH == "master" else VERSION
|
||||
docker_tag = "dev" if be.branch == "master" else be.version
|
||||
|
||||
click.echo("Uploading Docker image to tag={}...".format(docker_tag))
|
||||
subprocess.check_call([
|
||||
"docker",
|
||||
"login",
|
||||
"-u", os.environ["DOCKER_USERNAME"],
|
||||
"-p", os.environ["DOCKER_PASSWORD"],
|
||||
"-u", be.docker_username,
|
||||
"-p", be.docker_password,
|
||||
])
|
||||
subprocess.check_call([
|
||||
"docker",
|
||||
|
||||
95
test/release/test_cibuild.py
Normal file
95
test/release/test_cibuild.py
Normal file
@ -0,0 +1,95 @@
|
||||
import os
|
||||
import io
|
||||
from release import cibuild
|
||||
|
||||
|
||||
def test_buildenviron_common():
|
||||
be = cibuild.BuildEnviron(
|
||||
system = "Linux",
|
||||
root_dir = "/foo",
|
||||
|
||||
travis_tag = "v0.0.1",
|
||||
travis_branch = "v0.x",
|
||||
)
|
||||
assert be.release_dir == os.path.join(be.root_dir, "release")
|
||||
assert be.dist_dir == os.path.join(be.root_dir, "release", "dist")
|
||||
assert be.build_dir == os.path.join(be.root_dir, "release", "build")
|
||||
assert be.is_pull_request is False
|
||||
|
||||
cs = io.StringIO()
|
||||
be.dump_info(cs)
|
||||
assert cs.getvalue()
|
||||
|
||||
|
||||
def test_buildenviron_pr():
|
||||
be = cibuild.BuildEnviron(
|
||||
travis_tag = "v0.0.1",
|
||||
travis_branch = "v0.x",
|
||||
travis_pull_request = "true",
|
||||
)
|
||||
assert be.is_pull_request
|
||||
|
||||
be = cibuild.BuildEnviron(
|
||||
appveyor_pull_request_number = "xxxx",
|
||||
)
|
||||
assert be.is_pull_request
|
||||
|
||||
|
||||
def test_buildenviron_tag():
|
||||
be = cibuild.BuildEnviron(
|
||||
system = "Linux",
|
||||
root_dir = "/foo",
|
||||
|
||||
travis_tag = "v0.0.1",
|
||||
travis_branch = "v0.x",
|
||||
)
|
||||
assert be.tag == "v0.0.1"
|
||||
assert be.branch == "v0.x"
|
||||
assert be.version == "0.0.1"
|
||||
assert be.upload_dir == "0.0.1"
|
||||
|
||||
|
||||
def test_buildenviron_branch():
|
||||
be = cibuild.BuildEnviron(
|
||||
system = "Linux",
|
||||
root_dir = "/foo",
|
||||
|
||||
travis_tag = "",
|
||||
travis_branch = "v0.x",
|
||||
)
|
||||
assert be.tag == ""
|
||||
assert be.branch == "v0.x"
|
||||
assert be.version == "0.x"
|
||||
assert be.upload_dir == "branches/0.x"
|
||||
|
||||
|
||||
def test_buildenviron_osx():
|
||||
be = cibuild.BuildEnviron(
|
||||
system = "Darwin",
|
||||
root_dir = "/foo",
|
||||
|
||||
travis_tag = "v0.0.1",
|
||||
travis_branch = "v0.x",
|
||||
)
|
||||
assert be.platform_tag == "osx"
|
||||
assert be.bdists == {
|
||||
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
|
||||
"pathod": ["pathoc", "pathod"],
|
||||
}
|
||||
assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-osx.tar.gz"
|
||||
|
||||
|
||||
def test_buildenviron_windows():
|
||||
be = cibuild.BuildEnviron(
|
||||
system = "Windows",
|
||||
root_dir = "/foo",
|
||||
|
||||
travis_tag = "v0.0.1",
|
||||
travis_branch = "v0.x",
|
||||
)
|
||||
assert be.platform_tag == "windows"
|
||||
assert be.bdists == {
|
||||
"mitmproxy": ["mitmdump", "mitmweb"],
|
||||
"pathod": ["pathoc", "pathod"],
|
||||
}
|
||||
assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-windows.zip"
|
||||
Loading…
Reference in New Issue
Block a user