Merge pull request #3152 from mitmproxy/cibuild

Tame cibuild (somewhat)
This commit is contained in:
Aldo Cortesi 2018-05-25 08:29:24 +12:00 committed by GitHub
commit 66d612eeca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 438 additions and 164 deletions

0
release/__init__.py Normal file
View File

View File

@ -10,163 +10,263 @@ import shutil
import subprocess
import tarfile
import zipfile
from os.path import join, abspath, dirname, exists, basename
import click
import cryptography.fernet
# ZipFile and tarfile have slightly different APIs. Fix that.
if platform.system() == "Windows":
def Archive(name):
a = zipfile.ZipFile(name, "w")
a.add = a.write
return a
else:
def Archive(name):
return tarfile.open(name, "w:gz")
PLATFORM_TAG = {
"Darwin": "osx",
"Windows": "windows",
"Linux": "linux",
}.get(platform.system(), platform.system())
ROOT_DIR = abspath(join(dirname(__file__), ".."))
RELEASE_DIR = join(ROOT_DIR, "release")
BUILD_DIR = join(RELEASE_DIR, "build")
DIST_DIR = join(RELEASE_DIR, "dist")
BDISTS = {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"]
}
if platform.system() == "Windows":
BDISTS["mitmproxy"].remove("mitmproxy")
TOOLS = [
tool
for tools in sorted(BDISTS.values())
for tool in tools
]
TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
if TAG:
VERSION = re.sub('^v', '', TAG)
UPLOAD_DIR = VERSION
elif BRANCH:
VERSION = re.sub('^v', '', BRANCH)
UPLOAD_DIR = "branches/%s" % VERSION
else:
print("Could not establish build name - exiting." % BRANCH)
sys.exit(0)
print("BUILD PLATFORM_TAG=%s" % PLATFORM_TAG)
print("BUILD ROOT_DIR=%s" % ROOT_DIR)
print("BUILD RELEASE_DIR=%s" % RELEASE_DIR)
print("BUILD BUILD_DIR=%s" % BUILD_DIR)
print("BUILD DIST_DIR=%s" % DIST_DIR)
print("BUILD BDISTS=%s" % BDISTS)
print("BUILD TAG=%s" % TAG)
print("BUILD BRANCH=%s" % BRANCH)
print("BUILD VERSION=%s" % VERSION)
print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR)
def archive_name(bdist: str) -> str:
if platform.system() == "Windows":
ext = "zip"
else:
ext = "tar.gz"
return "{project}-{version}-{platform}.{ext}".format(
project=bdist,
version=VERSION,
platform=PLATFORM_TAG,
ext=ext
)
@contextlib.contextmanager
def chdir(path: str):
def chdir(path: str): # pragma: no cover
old_dir = os.getcwd()
os.chdir(path)
yield
os.chdir(old_dir)
@click.group(chain=True)
def cli():
"""
mitmproxy build tool
"""
class BuildError(Exception):
pass
@cli.command("build")
def build():
"""
Build a binary distribution
"""
os.makedirs(DIST_DIR, exist_ok=True)
class BuildEnviron:
PLATFORM_TAGS = {
"Darwin": "osx",
"Windows": "windows",
"Linux": "linux",
}
if "WHEEL" in os.environ:
whl = build_wheel()
else:
click.echo("Not building wheels.")
def __init__(
self,
*,
system = "",
root_dir = "",
if "WHEEL" in os.environ and "DOCKER" in os.environ:
# Docker image requires wheels
build_docker_image(whl)
else:
click.echo("Not building Docker image.")
travis_tag = "",
travis_branch = "",
travis_pull_request = "",
if "PYINSTALLER" in os.environ:
build_pyinstaller()
else:
click.echo("Not building PyInstaller packages.")
appveyor_repo_tag_name = "",
appveyor_repo_branch = "",
appveyor_pull_request_number = "",
should_build_wheel = False,
should_build_docker = False,
should_build_pyinstaller = False,
has_aws_creds = False,
has_twine_creds = False,
docker_username = "",
docker_password = "",
):
self.system = system
self.root_dir = root_dir
self.travis_tag = travis_tag
self.travis_branch = travis_branch
self.travis_pull_request = travis_pull_request
self.should_build_wheel = should_build_wheel
self.should_build_docker = should_build_docker
self.should_build_pyinstaller = should_build_pyinstaller
self.appveyor_repo_tag_name = appveyor_repo_tag_name
self.appveyor_repo_branch = appveyor_repo_branch
self.appveyor_pull_request_number = appveyor_pull_request_number
self.has_aws_creds = has_aws_creds
self.has_twine_creds = has_twine_creds
self.docker_username = docker_username
self.docker_password = docker_password
@classmethod
def from_env(klass):
return klass(
system = platform.system(),
root_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), "..")),
travis_tag = os.environ.get("TRAVIS_TAG", ""),
travis_branch = os.environ.get("TRAVIS_BRANCH", ""),
travis_pull_request = os.environ.get("TRAVIS_PULL_REQUEST"),
appveyor_repo_tag_name = os.environ.get("APPVEYOR_REPO_TAG_NAME", ""),
appveyor_repo_branch = os.environ.get("APPVEYOR_REPO_BRANCH", ""),
appveyor_pull_request_number = os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"),
should_build_wheel = "WHEEL" in os.environ,
should_build_pyinstaller = "PYINSTALLER" in os.environ,
should_build_docker = "DOCKER" in os.environ,
has_aws_creds = "AWS_ACCESS_KEY_ID" in os.environ,
has_twine_creds= (
"TWINE_USERNAME" in os.environ and
"TWINE_PASSWORD" in os.environ
),
docker_username = os.environ.get("DOCKER_USERNAME"),
docker_password = os.environ.get("DOCKER_PASSWORD"),
)
@property
def has_docker_creds(self) -> bool:
return self.docker_username and self.docker_password
@property
def is_pull_request(self) -> bool:
if self.appveyor_pull_request_number:
return True
if self.travis_pull_request and self.travis_pull_request != "false":
return True
return False
@property
def tag(self):
return self.travis_tag or self.appveyor_repo_tag_name
@property
def branch(self):
return self.travis_branch or self.appveyor_repo_branch
@property
def version(self):
name = self.tag or self.branch
if not name:
raise BuildError("We're on neither a tag nor a branch - could not establish version")
return re.sub('^v', "", name)
@property
def upload_dir(self):
if self.tag:
return self.version
else:
return "branches/%s" % self.version
@property
def platform_tag(self):
if self.system in self.PLATFORM_TAGS:
return self.PLATFORM_TAGS[self.system]
raise BuildError("Unsupported platform: %s" % self.system)
@property
def release_dir(self):
return os.path.join(self.root_dir, "release")
@property
def build_dir(self):
return os.path.join(self.release_dir, "build")
@property
def dist_dir(self):
return os.path.join(self.release_dir, "dist")
@property
def docker_tag(self):
if self.branch == "master":
return "dev"
return self.version
def archive(self, path):
# ZipFile and tarfile have slightly different APIs. Fix that.
if self.system == "Windows":
a = zipfile.ZipFile(path, "w")
a.add = a.write
return a
else:
return tarfile.open(path, "w:gz")
def archive_name(self, bdist: str) -> str:
if self.system == "Windows":
ext = "zip"
else:
ext = "tar.gz"
return "{project}-{version}-{platform}.{ext}".format(
project=bdist,
version=self.version,
platform=self.platform_tag,
ext=ext
)
@property
def bdists(self):
ret = {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"]
}
if self.system == "Windows":
ret["mitmproxy"].remove("mitmproxy")
return ret
@property
def should_upload_docker(self) -> bool:
return all([
(self.tag or self.branch == "master"),
self.should_build_docker,
self.has_docker_creds,
])
@property
def should_upload_pypi(self) -> bool:
if self.tag and self.should_build_wheel and self.has_twine_creds:
return True
return False
def dump_info(self, fp=sys.stdout):
lst = [
"version",
"tag",
"branch",
"platform_tag",
"root_dir",
"release_dir",
"build_dir",
"dist_dir",
"bdists",
"upload_dir",
"should_build_wheel",
"should_build_pyinstaller",
"should_build_docker",
"should_upload_docker",
"should_upload_pypi",
]
for attr in lst:
print("cibuild.%s=%s" % (attr, getattr(self, attr)), file=fp)
def build_wheel():
def build_wheel(be: BuildEnviron): # pragma: no cover
click.echo("Building wheel...")
subprocess.check_call([
"python",
"setup.py",
"-q",
"bdist_wheel",
"--dist-dir", DIST_DIR,
"--dist-dir", be.dist_dir,
])
whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
click.echo("Found wheel package: {}".format(whl))
subprocess.check_call([
"tox",
"-e", "wheeltest",
"--",
whl
])
subprocess.check_call(["tox", "-e", "wheeltest", "--", whl])
return whl
def build_docker_image(whl):
def build_docker_image(be: BuildEnviron, whl: str): # pragma: no cover
click.echo("Building Docker image...")
subprocess.check_call([
"docker",
"build",
"--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, ROOT_DIR)),
"--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(basename(whl)),
"--tag", "mitmproxy/mitmproxy/{}".format(be.docker_tag),
"--build-arg", "WHEEL_MITMPROXY={}".format(whl),
"--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(os.path.basename(whl)),
"--file", "docker/Dockerfile",
"."
])
def build_pyinstaller():
PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
def build_pyinstaller(be: BuildEnviron): # pragma: no cover
click.echo("Building pyinstaller package...")
PYINSTALLER_SPEC = os.path.join(be.release_dir, "specs")
# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
PYINSTALLER_HOOKS = os.path.abspath(os.path.join(be.release_dir, "hooks"))
PYINSTALLER_TEMP = os.path.abspath(os.path.join(be.build_dir, "pyinstaller"))
PYINSTALLER_DIST = os.path.abspath(os.path.join(be.build_dir, "binaries", be.platform_tag))
# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
@ -178,13 +278,13 @@ def build_pyinstaller():
else:
PYINSTALLER_ARGS = []
if exists(PYINSTALLER_TEMP):
if os.path.exists(PYINSTALLER_TEMP):
shutil.rmtree(PYINSTALLER_TEMP)
if exists(PYINSTALLER_DIST):
if os.path.exists(PYINSTALLER_DIST):
shutil.rmtree(PYINSTALLER_DIST)
for bdist, tools in sorted(BDISTS.items()):
with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
for bdist, tools in sorted(be.bdists.items()):
with be.archive(os.path.join(be.dist_dir, be.archive_name(bdist))) as archive:
for tool in tools:
# We can't have a folder and a file with the same name.
if tool == "mitmproxy":
@ -221,7 +321,7 @@ def build_pyinstaller():
os.remove("{}.spec".format(tool))
# Test if it works at all O:-)
executable = join(PYINSTALLER_DIST, tool)
executable = os.path.join(PYINSTALLER_DIST, tool)
if platform.system() == "Windows":
executable += ".exe"
@ -236,12 +336,39 @@ def build_pyinstaller():
click.echo("> %s --version" % executable)
click.echo(subprocess.check_output([executable, "--version"]).decode())
archive.add(executable, basename(executable))
click.echo("Packed {}.".format(archive_name(bdist)))
archive.add(executable, os.path.basename(executable))
click.echo("Packed {}.".format(be.archive_name(bdist)))
@click.group(chain=True)
def cli(): # pragma: no cover
"""
mitmproxy build tool
"""
pass
@cli.command("build")
def build(): # pragma: no cover
"""
Build a binary distribution
"""
be = BuildEnviron.from_env()
be.dump_info()
os.makedirs(be.dist_dir, exist_ok=True)
if be.should_build_wheel:
whl = build_wheel(be)
# Docker image requires wheels
if be.should_build_docker:
build_docker_image(be, whl)
if be.should_build_pyinstaller:
build_pyinstaller(be)
@cli.command("upload")
def upload():
def upload(): # pragma: no cover
"""
Upload build artifacts
@ -249,62 +376,38 @@ def upload():
Uploads the Pyinstaller and wheels packages to the snapshot server.
Pushes the Docker image to Docker Hub.
"""
be = BuildEnviron.from_env()
# Our credentials are only available from within the main repository and not forks.
# We need to prevent uploads from all BUT the branches in the main repository.
# Pull requests and master-branches of forks are not allowed to upload.
is_pull_request = (
("TRAVIS_PULL_REQUEST" in os.environ and os.environ["TRAVIS_PULL_REQUEST"] != "false") or
"APPVEYOR_PULL_REQUEST_NUMBER" in os.environ
)
if is_pull_request:
if be.is_pull_request:
click.echo("Refusing to upload artifacts from a pull request!")
return
if "AWS_ACCESS_KEY_ID" in os.environ:
if be.has_aws_creds:
subprocess.check_call([
"aws", "s3", "cp",
"--acl", "public-read",
DIST_DIR + "/",
"s3://snapshots.mitmproxy.org/{}/".format(UPLOAD_DIR),
be.dist_dir + "/",
"s3://snapshots.mitmproxy.org/{}/".format(be.upload_dir),
"--recursive",
])
upload_pypi = (
TAG and
"WHEEL" in os.environ and
"TWINE_USERNAME" in os.environ and
"TWINE_PASSWORD" in os.environ
)
if upload_pypi:
whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
if be.should_upload_pypi:
whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
click.echo("Uploading {} to PyPi...".format(whl))
subprocess.check_call([
"twine",
"upload",
whl
])
subprocess.check_call(["twine", "upload", whl])
upload_docker = (
(TAG or BRANCH == "master") and
"DOCKER" in os.environ and
"DOCKER_USERNAME" in os.environ and
"DOCKER_PASSWORD" in os.environ
)
if upload_docker:
docker_tag = "dev" if BRANCH == "master" else VERSION
click.echo("Uploading Docker image to tag={}...".format(docker_tag))
if be.should_upload_docker:
click.echo("Uploading Docker image to tag={}...".format(be.docker_tag))
subprocess.check_call([
"docker",
"login",
"-u", os.environ["DOCKER_USERNAME"],
"-p", os.environ["DOCKER_PASSWORD"],
"-u", be.docker_username,
"-p", be.docker_password,
])
subprocess.check_call([
"docker",
"push",
"mitmproxy/mitmproxy:{}".format(docker_tag),
"mitmproxy/mitmproxy:{}".format(be.docker_tag),
])
@ -312,10 +415,10 @@ def upload():
@click.argument('infile', type=click.File('rb'))
@click.argument('outfile', type=click.File('wb'))
@click.argument('key', envvar='RTOOL_KEY')
def decrypt(infile, outfile, key):
def decrypt(infile, outfile, key): # pragma: no cover
f = cryptography.fernet.Fernet(key.encode())
outfile.write(f.decrypt(infile.read()))
if __name__ == "__main__":
if __name__ == "__main__": # pragma: no cover
cli()

View File

@ -38,6 +38,8 @@ exclude =
pathod/pathod.py
pathod/test.py
pathod/protocols/http2.py
release/hooks
[tool:individual_coverage]
exclude =
@ -89,3 +91,4 @@ exclude =
pathod/protocols/http2.py
pathod/protocols/websockets.py
pathod/test.py
release/hooks

View File

@ -0,0 +1,168 @@
import os
import io
import pytest
from release import cibuild
def test_buildenviron_live():
be = cibuild.BuildEnviron.from_env()
assert be.release_dir
def test_buildenviron_common():
be = cibuild.BuildEnviron(
system = "Linux",
root_dir = "/foo",
travis_tag = "v0.0.1",
travis_branch = "v0.x",
)
assert be.release_dir == os.path.join(be.root_dir, "release")
assert be.dist_dir == os.path.join(be.root_dir, "release", "dist")
assert be.build_dir == os.path.join(be.root_dir, "release", "build")
assert be.is_pull_request is False
assert not be.has_docker_creds
cs = io.StringIO()
be.dump_info(cs)
assert cs.getvalue()
be = cibuild.BuildEnviron(
system = "Unknown",
root_dir = "/foo",
)
with pytest.raises(cibuild.BuildError):
be.version
with pytest.raises(cibuild.BuildError):
be.platform_tag
def test_buildenviron_pr():
# Simulates a PR. We build everything, but don't have access to secret
# credential env variables.
be = cibuild.BuildEnviron(
travis_tag = "",
travis_branch = "master",
travis_pull_request = "true",
should_build_wheel = True,
should_build_pyinstaller = True,
should_build_docker = True,
)
assert be.is_pull_request
# Mini test for appveyor
be = cibuild.BuildEnviron(
appveyor_pull_request_number = "xxxx",
)
assert be.is_pull_request
def test_buildenviron_commit():
# Simulates an ordinary commit on the master branch.
be = cibuild.BuildEnviron(
travis_tag = "",
travis_branch = "master",
travis_pull_request = "false",
should_build_wheel = True,
should_build_pyinstaller = True,
should_build_docker = True,
docker_username = "foo",
docker_password = "bar",
)
assert be.docker_tag == "dev"
assert be.should_upload_docker
assert not be.should_upload_pypi
assert be.should_upload_docker
def test_buildenviron_rleasetag():
# Simulates a tagged release on a release branch.
be = cibuild.BuildEnviron(
system = "Linux",
root_dir = "/foo",
travis_tag = "v0.0.1",
travis_branch = "v0.x",
should_build_wheel = True,
should_build_docker = True,
should_build_pyinstaller = True,
has_twine_creds = True,
docker_username = "foo",
docker_password = "bar",
)
assert be.tag == "v0.0.1"
assert be.branch == "v0.x"
assert be.version == "0.0.1"
assert be.upload_dir == "0.0.1"
assert be.docker_tag == "0.0.1"
assert be.should_upload_pypi
assert be.should_upload_docker
def test_buildenviron_branch():
# Simulates a development branch on the main repo
be = cibuild.BuildEnviron(
system = "Linux",
root_dir = "/foo",
travis_tag = "",
travis_branch = "mybranch",
should_build_wheel = True,
should_build_docker = True,
should_build_pyinstaller = True,
has_twine_creds = True,
docker_username = "foo",
docker_password = "bar",
)
assert be.tag == ""
assert be.branch == "mybranch"
assert be.version == "mybranch"
assert be.upload_dir == "branches/mybranch"
assert not be.should_upload_pypi
assert not be.should_upload_docker
def test_buildenviron_osx(tmpdir):
be = cibuild.BuildEnviron(
system = "Darwin",
root_dir = "/foo",
travis_tag = "v0.0.1",
travis_branch = "v0.x",
)
assert be.platform_tag == "osx"
assert be.bdists == {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"],
}
assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-osx.tar.gz"
a = be.archive(os.path.join(tmpdir, "arch"))
assert a
a.close()
def test_buildenviron_windows(tmpdir):
be = cibuild.BuildEnviron(
system = "Windows",
root_dir = "/foo",
travis_tag = "v0.0.1",
travis_branch = "v0.x",
)
assert be.platform_tag == "windows"
assert be.bdists == {
"mitmproxy": ["mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"],
}
assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-windows.zip"
a = be.archive(os.path.join(tmpdir, "arch"))
assert a
a.close()

View File

@ -12,7 +12,7 @@ setenv = HOME = {envtmpdir}
commands =
mitmdump --version
pytest --timeout 60 --cov-report='' \
--cov=mitmproxy --cov=pathod \
--cov=mitmproxy --cov=pathod --cov=release \
--full-cov=mitmproxy/ --full-cov=pathod/ \
{posargs}
{env:CI_COMMANDS:python -c ""}