From 8fceaca6b8a219cce6e42a05a72eb8cd80565c89 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 24 May 2018 11:06:05 +1200 Subject: [PATCH 1/6] cibuild: Consolidate build environment and add tests --- release/__init__.py | 0 release/cibuild.py | 370 +++++++++++++++++++++-------------- test/release/test_cibuild.py | 95 +++++++++ 3 files changed, 319 insertions(+), 146 deletions(-) create mode 100644 release/__init__.py create mode 100644 test/release/test_cibuild.py diff --git a/release/__init__.py b/release/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/release/cibuild.py b/release/cibuild.py index 3a79affe1..566c46594 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -15,76 +15,6 @@ from os.path import join, abspath, dirname, exists, basename import click import cryptography.fernet -# ZipFile and tarfile have slightly different APIs. Fix that. -if platform.system() == "Windows": - def Archive(name): - a = zipfile.ZipFile(name, "w") - a.add = a.write - return a -else: - def Archive(name): - return tarfile.open(name, "w:gz") - -PLATFORM_TAG = { - "Darwin": "osx", - "Windows": "windows", - "Linux": "linux", -}.get(platform.system(), platform.system()) - -ROOT_DIR = abspath(join(dirname(__file__), "..")) -RELEASE_DIR = join(ROOT_DIR, "release") -BUILD_DIR = join(RELEASE_DIR, "build") -DIST_DIR = join(RELEASE_DIR, "dist") - -BDISTS = { - "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathoc", "pathod"] -} -if platform.system() == "Windows": - BDISTS["mitmproxy"].remove("mitmproxy") - -TOOLS = [ - tool - for tools in sorted(BDISTS.values()) - for tool in tools -] - -TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None)) -BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None)) -if TAG: - VERSION = re.sub('^v', '', TAG) - UPLOAD_DIR = VERSION -elif BRANCH: - VERSION = re.sub('^v', '', BRANCH) - UPLOAD_DIR = "branches/%s" % VERSION -else: - print("Could not establish build name - exiting." % BRANCH) - sys.exit(0) - -print("BUILD PLATFORM_TAG=%s" % PLATFORM_TAG) -print("BUILD ROOT_DIR=%s" % ROOT_DIR) -print("BUILD RELEASE_DIR=%s" % RELEASE_DIR) -print("BUILD BUILD_DIR=%s" % BUILD_DIR) -print("BUILD DIST_DIR=%s" % DIST_DIR) -print("BUILD BDISTS=%s" % BDISTS) -print("BUILD TAG=%s" % TAG) -print("BUILD BRANCH=%s" % BRANCH) -print("BUILD VERSION=%s" % VERSION) -print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR) - - -def archive_name(bdist: str) -> str: - if platform.system() == "Windows": - ext = "zip" - else: - ext = "tar.gz" - return "{project}-{version}-{platform}.{ext}".format( - project=bdist, - version=VERSION, - platform=PLATFORM_TAG, - ext=ext - ) - @contextlib.contextmanager def chdir(path: str): @@ -94,79 +24,216 @@ def chdir(path: str): os.chdir(old_dir) -@click.group(chain=True) -def cli(): - """ - mitmproxy build tool - """ +class BuildError(Exception): pass -@cli.command("build") -def build(): - """ - Build a binary distribution - """ - os.makedirs(DIST_DIR, exist_ok=True) +class BuildEnviron: + PLATFORM_TAGS = { + "Darwin": "osx", + "Windows": "windows", + "Linux": "linux", + } - if "WHEEL" in os.environ: - whl = build_wheel() - else: - click.echo("Not building wheels.") + def __init__( + self, + *, + system = "", + root_dir = "", - if "WHEEL" in os.environ and "DOCKER" in os.environ: - # Docker image requires wheels - build_docker_image(whl) - else: - click.echo("Not building Docker image.") + travis_tag = "", + travis_branch = "", + travis_pull_request = "", - if "PYINSTALLER" in os.environ: - build_pyinstaller() - else: - click.echo("Not building PyInstaller packages.") + appveyor_repo_tag_name = "", + appveyor_repo_branch = "", + appveyor_pull_request_number = "", + + build_wheel = "", + build_docker = "", + build_pyinstaller = "", + + has_aws_creds = False, + has_twine_creds = False, + + docker_username = "", + docker_password = "", + ): + self.system = system + self.root_dir = root_dir + + self.travis_tag = travis_tag + self.travis_branch = travis_branch + self.travis_pull_request = travis_pull_request + + self.appveyor_repo_tag_name = appveyor_repo_tag_name + self.appveyor_repo_branch = appveyor_repo_branch + self.appveyor_pull_request_number = appveyor_pull_request_number + + self.has_aws_creds = has_aws_creds + self.has_twine_creds = has_twine_creds + self.docker_username = docker_username + self.docker_password = docker_password + + @classmethod + def from_env(klass): + return klass( + system = platform.system, + root_dir = dirname(__file__), + + travis_tag = os.environ.get("TRAVIS_TAG", ""), + travis_branch = os.environ.get("TRAVIS_BRANCH", ""), + travis_pull_request = os.environ.get("TRAVIS_PULL_REQUEST"), + + appveyor_repo_tag_name = os.environ.get("APPVEYOR_REPO_TAG_NAME", ""), + appveyor_repo_branch = os.environ.get("APPVEYOR_REPO_BRANCH", ""), + appveyor_pull_request_number = os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"), + + build_wheel = "WHEEL" in os.environ, + build_pyinstaller = "PYINSTALLER" in os.environ, + build_docker = "DOCKER" in os.environ, + + has_aws_creds = "AWS_ACCESS_KEY_ID" in os.environ, + has_twine_creds= ( + "TWINE_USERNAME" in os.environ and + "TWINE_PASSWORD" in os.environ + ), + + docker_username = os.environ.get("DOCKER_USERNAME"), + docker_password = os.environ.get("DOCKER_PASSWORD"), + ) + + @property + def has_docker_creds(self) -> bool: + return self.docker_username and self.docker_password + + @property + def is_pull_request(self) -> bool: + if self.appveyor_pull_request_number: + return True + if self.travis_pull_request and self.travis_pull_request != "false": + return True + return False + + @property + def tag(self): + return self.travis_tag or self.appveyor_repo_tag_name + + @property + def branch(self): + return self.travis_branch or self.appveyor_repo_branch + + @property + def version(self): + name = self.tag or self.branch + if not name: + raise BuildError("Could not establish build name") + return re.sub('^v', "", name) + + @property + def upload_dir(self): + if self.tag: + return self.version + else: + return "branches/%s" % self.version + + @property + def platform_tag(self): + if self.system in self.PLATFORM_TAGS: + return self.PLATFORM_TAGS[self.system] + raise BuildError("Unsupported platform: %s" % self.system) + + @property + def release_dir(self): + return os.path.join(self.root_dir, "release") + + @property + def build_dir(self): + return os.path.join(self.release_dir, "build") + + @property + def dist_dir(self): + return os.path.join(self.release_dir, "dist") + + def archive(self, name): + # ZipFile and tarfile have slightly different APIs. Fix that. + if self.system == "Windows": + a = zipfile.ZipFile(name, "w") + a.add = a.write + return a + else: + return tarfile.open(name, "w:gz") + + def archive_name(self, bdist: str) -> str: + if self.system == "Windows": + ext = "zip" + else: + ext = "tar.gz" + return "{project}-{version}-{platform}.{ext}".format( + project=bdist, + version=self.version, + platform=self.platform_tag, + ext=ext + ) + + @property + def bdists(self): + ret = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"] + } + if self.system == "Windows": + ret["mitmproxy"].remove("mitmproxy") + return ret + + def dump_info(self, fp=sys.stdout): + print("BUILD PLATFORM_TAG=%s" % self.platform_tag, file=fp) + print("BUILD ROOT_DIR=%s" % self.root_dir, file=fp) + print("BUILD RELEASE_DIR=%s" % self.release_dir, file=fp) + print("BUILD BUILD_DIR=%s" % self.build_dir, file=fp) + print("BUILD DIST_DIR=%s" % self.dist_dir, file=fp) + print("BUILD BDISTS=%s" % self.bdists, file=fp) + print("BUILD TAG=%s" % self.tag, file=fp) + print("BUILD BRANCH=%s" % self.branch, file=fp) + print("BUILD VERSION=%s" % self.version, file=fp) + print("BUILD UPLOAD_DIR=%s" % self.upload_dir, file=fp) -def build_wheel(): +def build_wheel(be: BuildEnviron): click.echo("Building wheel...") subprocess.check_call([ "python", "setup.py", "-q", "bdist_wheel", - "--dist-dir", DIST_DIR, + "--dist-dir", be.dist_dir, ]) - - whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0] + whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0] click.echo("Found wheel package: {}".format(whl)) - - subprocess.check_call([ - "tox", - "-e", "wheeltest", - "--", - whl - ]) - + subprocess.check_call(["tox", "-e", "wheeltest", "--", whl]) return whl -def build_docker_image(whl): +def build_docker_image(be: BuildEnviron, whl: str): click.echo("Building Docker image...") subprocess.check_call([ "docker", "build", - "--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, ROOT_DIR)), + "--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, be.root_dir)), "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(basename(whl)), "--file", "docker/Dockerfile", "." ]) -def build_pyinstaller(): - PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") +def build_pyinstaller(be: BuildEnviron): + click.echo("Building pyinstaller package...") + + PYINSTALLER_SPEC = join(be.release_dir, "specs") # PyInstaller 3.2 does not bundle pydivert's Windivert binaries - PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") - PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") - PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) + PYINSTALLER_HOOKS = join(be.release_dir, "hooks") + PYINSTALLER_TEMP = join(be.build_dir, "pyinstaller") + PYINSTALLER_DIST = join(be.build_dir, "binaries", be.platform_tag) # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ @@ -183,8 +250,8 @@ def build_pyinstaller(): if exists(PYINSTALLER_DIST): shutil.rmtree(PYINSTALLER_DIST) - for bdist, tools in sorted(BDISTS.items()): - with Archive(join(DIST_DIR, archive_name(bdist))) as archive: + for bdist, tools in sorted(be.bdists.items()): + with be.archive(join(be.dist_dir, be.archive_name(bdist))) as archive: for tool in tools: # We can't have a folder and a file with the same name. if tool == "mitmproxy": @@ -237,7 +304,34 @@ def build_pyinstaller(): click.echo(subprocess.check_output([executable, "--version"]).decode()) archive.add(executable, basename(executable)) - click.echo("Packed {}.".format(archive_name(bdist))) + click.echo("Packed {}.".format(be.archive_name(bdist))) + + +@click.group(chain=True) +def cli(): + """ + mitmproxy build tool + """ + pass + + +@cli.command("build") +def build(): + """ + Build a binary distribution + """ + be = BuildEnviron.from_env() + be.dump_info() + + os.makedirs(be.dist_dir, exist_ok=True) + + if be.build_wheel: + whl = build_wheel(be) + # Docker image requires wheels + if be.build_docker: + build_docker_image(whl) + if be.build_pyinstaller: + build_pyinstaller() @cli.command("upload") @@ -249,57 +343,41 @@ def upload(): Uploads the Pyinstaller and wheels packages to the snapshot server. Pushes the Docker image to Docker Hub. """ + be = BuildEnviron.from_env() - # Our credentials are only available from within the main repository and not forks. - # We need to prevent uploads from all BUT the branches in the main repository. - # Pull requests and master-branches of forks are not allowed to upload. - is_pull_request = ( - ("TRAVIS_PULL_REQUEST" in os.environ and os.environ["TRAVIS_PULL_REQUEST"] != "false") or - "APPVEYOR_PULL_REQUEST_NUMBER" in os.environ - ) - if is_pull_request: + if be.is_pull_request: click.echo("Refusing to upload artifacts from a pull request!") return - if "AWS_ACCESS_KEY_ID" in os.environ: + if be.has_aws_creds: subprocess.check_call([ "aws", "s3", "cp", "--acl", "public-read", - DIST_DIR + "/", - "s3://snapshots.mitmproxy.org/{}/".format(UPLOAD_DIR), + be.dist_dir + "/", + "s3://snapshots.mitmproxy.org/{}/".format(be.upload_dir), "--recursive", ]) - upload_pypi = ( - TAG and - "WHEEL" in os.environ and - "TWINE_USERNAME" in os.environ and - "TWINE_PASSWORD" in os.environ - ) + upload_pypi = (be.tag and be.build_wheel and be.has_twine_creds) if upload_pypi: - whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0] + whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0] click.echo("Uploading {} to PyPi...".format(whl)) - subprocess.check_call([ - "twine", - "upload", - whl - ]) + subprocess.check_call(["twine", "upload", whl]) upload_docker = ( - (TAG or BRANCH == "master") and - "DOCKER" in os.environ and - "DOCKER_USERNAME" in os.environ and - "DOCKER_PASSWORD" in os.environ + (be.tag or be.branch == "master") and + be.build_docker, + be.has_docker_creds, ) if upload_docker: - docker_tag = "dev" if BRANCH == "master" else VERSION + docker_tag = "dev" if be.branch == "master" else be.version click.echo("Uploading Docker image to tag={}...".format(docker_tag)) subprocess.check_call([ "docker", "login", - "-u", os.environ["DOCKER_USERNAME"], - "-p", os.environ["DOCKER_PASSWORD"], + "-u", be.docker_username, + "-p", be.docker_password, ]) subprocess.check_call([ "docker", diff --git a/test/release/test_cibuild.py b/test/release/test_cibuild.py new file mode 100644 index 000000000..51aa0f7eb --- /dev/null +++ b/test/release/test_cibuild.py @@ -0,0 +1,95 @@ +import os +import io +from release import cibuild + + +def test_buildenviron_common(): + be = cibuild.BuildEnviron( + system = "Linux", + root_dir = "/foo", + + travis_tag = "v0.0.1", + travis_branch = "v0.x", + ) + assert be.release_dir == os.path.join(be.root_dir, "release") + assert be.dist_dir == os.path.join(be.root_dir, "release", "dist") + assert be.build_dir == os.path.join(be.root_dir, "release", "build") + assert be.is_pull_request is False + + cs = io.StringIO() + be.dump_info(cs) + assert cs.getvalue() + + +def test_buildenviron_pr(): + be = cibuild.BuildEnviron( + travis_tag = "v0.0.1", + travis_branch = "v0.x", + travis_pull_request = "true", + ) + assert be.is_pull_request + + be = cibuild.BuildEnviron( + appveyor_pull_request_number = "xxxx", + ) + assert be.is_pull_request + + +def test_buildenviron_tag(): + be = cibuild.BuildEnviron( + system = "Linux", + root_dir = "/foo", + + travis_tag = "v0.0.1", + travis_branch = "v0.x", + ) + assert be.tag == "v0.0.1" + assert be.branch == "v0.x" + assert be.version == "0.0.1" + assert be.upload_dir == "0.0.1" + + +def test_buildenviron_branch(): + be = cibuild.BuildEnviron( + system = "Linux", + root_dir = "/foo", + + travis_tag = "", + travis_branch = "v0.x", + ) + assert be.tag == "" + assert be.branch == "v0.x" + assert be.version == "0.x" + assert be.upload_dir == "branches/0.x" + + +def test_buildenviron_osx(): + be = cibuild.BuildEnviron( + system = "Darwin", + root_dir = "/foo", + + travis_tag = "v0.0.1", + travis_branch = "v0.x", + ) + assert be.platform_tag == "osx" + assert be.bdists == { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"], + } + assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-osx.tar.gz" + + +def test_buildenviron_windows(): + be = cibuild.BuildEnviron( + system = "Windows", + root_dir = "/foo", + + travis_tag = "v0.0.1", + travis_branch = "v0.x", + ) + assert be.platform_tag == "windows" + assert be.bdists == { + "mitmproxy": ["mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"], + } + assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-windows.zip" \ No newline at end of file From bae4a3393f4fd9cccc229e4b9c3d997fe7cfb4c1 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 24 May 2018 17:10:50 +1200 Subject: [PATCH 2/6] release: include release directory in coverage With lots of exclusions for now --- release/cibuild.py | 71 +++++++++++++++++++----------------- setup.cfg | 3 ++ test/release/test_cibuild.py | 32 ++++++++++++++-- tox.ini | 2 +- 4 files changed, 70 insertions(+), 38 deletions(-) diff --git a/release/cibuild.py b/release/cibuild.py index 566c46594..e2ac88d31 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -10,14 +10,13 @@ import shutil import subprocess import tarfile import zipfile -from os.path import join, abspath, dirname, exists, basename import click import cryptography.fernet @contextlib.contextmanager -def chdir(path: str): +def chdir(path: str): # pragma: no cover old_dir = os.getcwd() os.chdir(path) yield @@ -49,9 +48,9 @@ class BuildEnviron: appveyor_repo_branch = "", appveyor_pull_request_number = "", - build_wheel = "", - build_docker = "", - build_pyinstaller = "", + build_wheel = False, + build_docker = False, + build_pyinstaller = False, has_aws_creds = False, has_twine_creds = False, @@ -66,6 +65,10 @@ class BuildEnviron: self.travis_branch = travis_branch self.travis_pull_request = travis_pull_request + self.build_wheel = build_wheel + self.build_docker = build_docker + self.build_pyinstaller = build_pyinstaller + self.appveyor_repo_tag_name = appveyor_repo_tag_name self.appveyor_repo_branch = appveyor_repo_branch self.appveyor_pull_request_number = appveyor_pull_request_number @@ -78,8 +81,8 @@ class BuildEnviron: @classmethod def from_env(klass): return klass( - system = platform.system, - root_dir = dirname(__file__), + system = platform.system(), + root_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), "..")), travis_tag = os.environ.get("TRAVIS_TAG", ""), travis_branch = os.environ.get("TRAVIS_BRANCH", ""), @@ -127,7 +130,7 @@ class BuildEnviron: def version(self): name = self.tag or self.branch if not name: - raise BuildError("Could not establish build name") + raise BuildError("We're on neither a tag nor a branch - could not establish version") return re.sub('^v', "", name) @property @@ -155,14 +158,14 @@ class BuildEnviron: def dist_dir(self): return os.path.join(self.release_dir, "dist") - def archive(self, name): + def archive(self, path): # ZipFile and tarfile have slightly different APIs. Fix that. if self.system == "Windows": - a = zipfile.ZipFile(name, "w") + a = zipfile.ZipFile(path, "w") a.add = a.write return a else: - return tarfile.open(name, "w:gz") + return tarfile.open(path, "w:gz") def archive_name(self, bdist: str) -> str: if self.system == "Windows": @@ -199,7 +202,7 @@ class BuildEnviron: print("BUILD UPLOAD_DIR=%s" % self.upload_dir, file=fp) -def build_wheel(be: BuildEnviron): +def build_wheel(be: BuildEnviron): # pragma: no cover click.echo("Building wheel...") subprocess.check_call([ "python", @@ -208,32 +211,32 @@ def build_wheel(be: BuildEnviron): "bdist_wheel", "--dist-dir", be.dist_dir, ]) - whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0] + whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0] click.echo("Found wheel package: {}".format(whl)) subprocess.check_call(["tox", "-e", "wheeltest", "--", whl]) return whl -def build_docker_image(be: BuildEnviron, whl: str): +def build_docker_image(be: BuildEnviron, whl: str): # pragma: no cover click.echo("Building Docker image...") subprocess.check_call([ "docker", "build", - "--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, be.root_dir)), - "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(basename(whl)), + "--build-arg", "WHEEL_MITMPROXY={}".format(whl), + "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(os.path.basename(whl)), "--file", "docker/Dockerfile", "." ]) -def build_pyinstaller(be: BuildEnviron): +def build_pyinstaller(be: BuildEnviron): # pragma: no cover click.echo("Building pyinstaller package...") - PYINSTALLER_SPEC = join(be.release_dir, "specs") + PYINSTALLER_SPEC = os.path.join(be.release_dir, "specs") # PyInstaller 3.2 does not bundle pydivert's Windivert binaries - PYINSTALLER_HOOKS = join(be.release_dir, "hooks") - PYINSTALLER_TEMP = join(be.build_dir, "pyinstaller") - PYINSTALLER_DIST = join(be.build_dir, "binaries", be.platform_tag) + PYINSTALLER_HOOKS = os.path.abspath(os.path.join(be.release_dir, "hooks")) + PYINSTALLER_TEMP = os.path.abspath(os.path.join(be.build_dir, "pyinstaller")) + PYINSTALLER_DIST = os.path.abspath(os.path.join(be.build_dir, "binaries", be.platform_tag)) # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ @@ -245,13 +248,13 @@ def build_pyinstaller(be: BuildEnviron): else: PYINSTALLER_ARGS = [] - if exists(PYINSTALLER_TEMP): + if os.path.exists(PYINSTALLER_TEMP): shutil.rmtree(PYINSTALLER_TEMP) - if exists(PYINSTALLER_DIST): + if os.path.exists(PYINSTALLER_DIST): shutil.rmtree(PYINSTALLER_DIST) for bdist, tools in sorted(be.bdists.items()): - with be.archive(join(be.dist_dir, be.archive_name(bdist))) as archive: + with be.archive(os.path.join(be.dist_dir, be.archive_name(bdist))) as archive: for tool in tools: # We can't have a folder and a file with the same name. if tool == "mitmproxy": @@ -288,7 +291,7 @@ def build_pyinstaller(be: BuildEnviron): os.remove("{}.spec".format(tool)) # Test if it works at all O:-) - executable = join(PYINSTALLER_DIST, tool) + executable = os.path.join(PYINSTALLER_DIST, tool) if platform.system() == "Windows": executable += ".exe" @@ -303,12 +306,12 @@ def build_pyinstaller(be: BuildEnviron): click.echo("> %s --version" % executable) click.echo(subprocess.check_output([executable, "--version"]).decode()) - archive.add(executable, basename(executable)) + archive.add(executable, os.path.basename(executable)) click.echo("Packed {}.".format(be.archive_name(bdist))) @click.group(chain=True) -def cli(): +def cli(): # pragma: no cover """ mitmproxy build tool """ @@ -316,7 +319,7 @@ def cli(): @cli.command("build") -def build(): +def build(): # pragma: no cover """ Build a binary distribution """ @@ -329,13 +332,13 @@ def build(): whl = build_wheel(be) # Docker image requires wheels if be.build_docker: - build_docker_image(whl) + build_docker_image(be, whl) if be.build_pyinstaller: - build_pyinstaller() + build_pyinstaller(be) @cli.command("upload") -def upload(): +def upload(): # pragma: no cover """ Upload build artifacts @@ -360,7 +363,7 @@ def upload(): upload_pypi = (be.tag and be.build_wheel and be.has_twine_creds) if upload_pypi: - whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0] + whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0] click.echo("Uploading {} to PyPi...".format(whl)) subprocess.check_call(["twine", "upload", whl]) @@ -390,10 +393,10 @@ def upload(): @click.argument('infile', type=click.File('rb')) @click.argument('outfile', type=click.File('wb')) @click.argument('key', envvar='RTOOL_KEY') -def decrypt(infile, outfile, key): +def decrypt(infile, outfile, key): # pragma: no cover f = cryptography.fernet.Fernet(key.encode()) outfile.write(f.decrypt(infile.read())) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover cli() diff --git a/setup.cfg b/setup.cfg index 592cc2e33..32b6aac3c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,6 +38,8 @@ exclude = pathod/pathod.py pathod/test.py pathod/protocols/http2.py + release/hooks + [tool:individual_coverage] exclude = @@ -89,3 +91,4 @@ exclude = pathod/protocols/http2.py pathod/protocols/websockets.py pathod/test.py + release/hooks diff --git a/test/release/test_cibuild.py b/test/release/test_cibuild.py index 51aa0f7eb..23b8e6cd3 100644 --- a/test/release/test_cibuild.py +++ b/test/release/test_cibuild.py @@ -1,8 +1,16 @@ import os import io + +import pytest + from release import cibuild +def test_buildenviron_live(): + be = cibuild.BuildEnviron.from_env() + assert be.release_dir + + def test_buildenviron_common(): be = cibuild.BuildEnviron( system = "Linux", @@ -15,11 +23,21 @@ def test_buildenviron_common(): assert be.dist_dir == os.path.join(be.root_dir, "release", "dist") assert be.build_dir == os.path.join(be.root_dir, "release", "build") assert be.is_pull_request is False + assert not be.has_docker_creds cs = io.StringIO() be.dump_info(cs) assert cs.getvalue() + be = cibuild.BuildEnviron( + system = "Unknown", + root_dir = "/foo", + ) + with pytest.raises(cibuild.BuildError): + be.version + with pytest.raises(cibuild.BuildError): + be.platform_tag + def test_buildenviron_pr(): be = cibuild.BuildEnviron( @@ -63,7 +81,7 @@ def test_buildenviron_branch(): assert be.upload_dir == "branches/0.x" -def test_buildenviron_osx(): +def test_buildenviron_osx(tmpdir): be = cibuild.BuildEnviron( system = "Darwin", root_dir = "/foo", @@ -78,8 +96,12 @@ def test_buildenviron_osx(): } assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-osx.tar.gz" + a = be.archive(os.path.join(tmpdir, "arch")) + assert a + a.close() -def test_buildenviron_windows(): + +def test_buildenviron_windows(tmpdir): be = cibuild.BuildEnviron( system = "Windows", root_dir = "/foo", @@ -92,4 +114,8 @@ def test_buildenviron_windows(): "mitmproxy": ["mitmdump", "mitmweb"], "pathod": ["pathoc", "pathod"], } - assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-windows.zip" \ No newline at end of file + assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-windows.zip" + + a = be.archive(os.path.join(tmpdir, "arch")) + assert a + a.close() \ No newline at end of file diff --git a/tox.ini b/tox.ini index 64dfa78ed..812f1a4b3 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ setenv = HOME = {envtmpdir} commands = mitmdump --version pytest --timeout 60 --cov-report='' \ - --cov=mitmproxy --cov=pathod \ + --cov=mitmproxy --cov=pathod --cov=release \ --full-cov=mitmproxy/ --full-cov=pathod/ \ {posargs} {env:CI_COMMANDS:python -c ""} From 0afff3a952e85401c41efc0e157ab07d3b985061 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 24 May 2018 21:59:45 +1200 Subject: [PATCH 3/6] cibuild: add docker tag calculation to environ --- release/cibuild.py | 13 +++++++++---- test/release/test_cibuild.py | 9 +++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/release/cibuild.py b/release/cibuild.py index e2ac88d31..491617fa3 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -158,6 +158,12 @@ class BuildEnviron: def dist_dir(self): return os.path.join(self.release_dir, "dist") + @property + def docker_tag(self): + if self.branch == "master": + return "dev" + return self.version + def archive(self, path): # ZipFile and tarfile have slightly different APIs. Fix that. if self.system == "Windows": @@ -222,6 +228,7 @@ def build_docker_image(be: BuildEnviron, whl: str): # pragma: no cover subprocess.check_call([ "docker", "build", + "--tag", be.docker_tag, "--build-arg", "WHEEL_MITMPROXY={}".format(whl), "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(os.path.basename(whl)), "--file", "docker/Dockerfile", @@ -373,9 +380,7 @@ def upload(): # pragma: no cover be.has_docker_creds, ) if upload_docker: - docker_tag = "dev" if be.branch == "master" else be.version - - click.echo("Uploading Docker image to tag={}...".format(docker_tag)) + click.echo("Uploading Docker image to tag={}...".format(be.docker_tag)) subprocess.check_call([ "docker", "login", @@ -385,7 +390,7 @@ def upload(): # pragma: no cover subprocess.check_call([ "docker", "push", - "mitmproxy/mitmproxy:{}".format(docker_tag), + "mitmproxy/mitmproxy:{}".format(be.docker_tag), ]) diff --git a/test/release/test_cibuild.py b/test/release/test_cibuild.py index 23b8e6cd3..a60f101ff 100644 --- a/test/release/test_cibuild.py +++ b/test/release/test_cibuild.py @@ -53,6 +53,14 @@ def test_buildenviron_pr(): assert be.is_pull_request +def test_buildenviron_commit(): + be = cibuild.BuildEnviron( + travis_branch = "master", + travis_pull_request = "false", + ) + assert be.docker_tag == "dev" + + def test_buildenviron_tag(): be = cibuild.BuildEnviron( system = "Linux", @@ -65,6 +73,7 @@ def test_buildenviron_tag(): assert be.branch == "v0.x" assert be.version == "0.0.1" assert be.upload_dir == "0.0.1" + assert be.docker_tag == "0.0.1" def test_buildenviron_branch(): From c91b511bc7cca6a04237d30db1845ddc232aff64 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 24 May 2018 22:15:55 +1200 Subject: [PATCH 4/6] cibuild: extract upload conditions --- release/cibuild.py | 50 +++++++++++++++++++++--------------- test/release/test_cibuild.py | 9 ++++++- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/release/cibuild.py b/release/cibuild.py index 491617fa3..f31012699 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -48,9 +48,9 @@ class BuildEnviron: appveyor_repo_branch = "", appveyor_pull_request_number = "", - build_wheel = False, - build_docker = False, - build_pyinstaller = False, + should_build_wheel = False, + should_build_docker = False, + should_build_pyinstaller = False, has_aws_creds = False, has_twine_creds = False, @@ -65,9 +65,9 @@ class BuildEnviron: self.travis_branch = travis_branch self.travis_pull_request = travis_pull_request - self.build_wheel = build_wheel - self.build_docker = build_docker - self.build_pyinstaller = build_pyinstaller + self.should_build_wheel = should_build_wheel + self.should_build_docker = should_build_docker + self.should_build_pyinstaller = should_build_pyinstaller self.appveyor_repo_tag_name = appveyor_repo_tag_name self.appveyor_repo_branch = appveyor_repo_branch @@ -92,9 +92,9 @@ class BuildEnviron: appveyor_repo_branch = os.environ.get("APPVEYOR_REPO_BRANCH", ""), appveyor_pull_request_number = os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"), - build_wheel = "WHEEL" in os.environ, - build_pyinstaller = "PYINSTALLER" in os.environ, - build_docker = "DOCKER" in os.environ, + should_build_wheel = "WHEEL" in os.environ, + should_build_pyinstaller = "PYINSTALLER" in os.environ, + should_build_docker = "DOCKER" in os.environ, has_aws_creds = "AWS_ACCESS_KEY_ID" in os.environ, has_twine_creds= ( @@ -195,6 +195,20 @@ class BuildEnviron: ret["mitmproxy"].remove("mitmproxy") return ret + @property + def should_upload_docker(self) -> bool: + return ( + (self.tag or self.branch == "master") and + self.should_build_docker, + self.has_docker_creds, + ) + + @property + def should_upload_pypi(self) -> bool: + if self.tag and self.should_build_wheel and self.has_twine_creds: + return True + return False + def dump_info(self, fp=sys.stdout): print("BUILD PLATFORM_TAG=%s" % self.platform_tag, file=fp) print("BUILD ROOT_DIR=%s" % self.root_dir, file=fp) @@ -228,7 +242,7 @@ def build_docker_image(be: BuildEnviron, whl: str): # pragma: no cover subprocess.check_call([ "docker", "build", - "--tag", be.docker_tag, + "--tag", "mitmproxy/mitmproxy/{}".format(be.docker_tag), "--build-arg", "WHEEL_MITMPROXY={}".format(whl), "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(os.path.basename(whl)), "--file", "docker/Dockerfile", @@ -335,12 +349,12 @@ def build(): # pragma: no cover os.makedirs(be.dist_dir, exist_ok=True) - if be.build_wheel: + if be.should_build_wheel: whl = build_wheel(be) # Docker image requires wheels - if be.build_docker: + if be.should_build_docker: build_docker_image(be, whl) - if be.build_pyinstaller: + if be.should_build_pyinstaller: build_pyinstaller(be) @@ -368,18 +382,12 @@ def upload(): # pragma: no cover "--recursive", ]) - upload_pypi = (be.tag and be.build_wheel and be.has_twine_creds) - if upload_pypi: + if be.should_upload_pypi: whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0] click.echo("Uploading {} to PyPi...".format(whl)) subprocess.check_call(["twine", "upload", whl]) - upload_docker = ( - (be.tag or be.branch == "master") and - be.build_docker, - be.has_docker_creds, - ) - if upload_docker: + if be.should_upload_docker: click.echo("Uploading Docker image to tag={}...".format(be.docker_tag)) subprocess.check_call([ "docker", diff --git a/test/release/test_cibuild.py b/test/release/test_cibuild.py index a60f101ff..2df5b4a09 100644 --- a/test/release/test_cibuild.py +++ b/test/release/test_cibuild.py @@ -59,21 +59,28 @@ def test_buildenviron_commit(): travis_pull_request = "false", ) assert be.docker_tag == "dev" + assert be.should_upload_docker + assert not be.should_upload_pypi -def test_buildenviron_tag(): +def test_buildenviron_rleasetag(): be = cibuild.BuildEnviron( system = "Linux", root_dir = "/foo", travis_tag = "v0.0.1", travis_branch = "v0.x", + should_build_wheel = True, + should_build_docker = True, + should_build_pyinstaller = True, + has_twine_creds = True, ) assert be.tag == "v0.0.1" assert be.branch == "v0.x" assert be.version == "0.0.1" assert be.upload_dir == "0.0.1" assert be.docker_tag == "0.0.1" + assert be.should_upload_pypi def test_buildenviron_branch(): From e9b19dba8aa204ead6092fbb901dadb5be8df72a Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 24 May 2018 22:32:31 +1200 Subject: [PATCH 5/6] cibuild: fix docker upload condition, expand tests --- release/cibuild.py | 6 +++--- test/release/test_cibuild.py | 39 ++++++++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/release/cibuild.py b/release/cibuild.py index f31012699..9d99d8d9c 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -197,11 +197,11 @@ class BuildEnviron: @property def should_upload_docker(self) -> bool: - return ( - (self.tag or self.branch == "master") and + return all([ + (self.tag or self.branch == "master"), self.should_build_docker, self.has_docker_creds, - ) + ]) @property def should_upload_pypi(self) -> bool: diff --git a/test/release/test_cibuild.py b/test/release/test_cibuild.py index 2df5b4a09..2b469d1be 100644 --- a/test/release/test_cibuild.py +++ b/test/release/test_cibuild.py @@ -40,13 +40,20 @@ def test_buildenviron_common(): def test_buildenviron_pr(): + # Simulates a PR. We build everything, but don't have access to secret + # credential env variables. be = cibuild.BuildEnviron( travis_tag = "v0.0.1", travis_branch = "v0.x", travis_pull_request = "true", + + should_build_wheel = True, + should_build_pyinstaller = True, + should_build_docker = True, ) assert be.is_pull_request + # Mini test for appveyor be = cibuild.BuildEnviron( appveyor_pull_request_number = "xxxx", ) @@ -54,26 +61,39 @@ def test_buildenviron_pr(): def test_buildenviron_commit(): + # Simulates an ordinary commit on the master branch. be = cibuild.BuildEnviron( + travis_tag = "", travis_branch = "master", travis_pull_request = "false", + + should_build_wheel = True, + should_build_pyinstaller = True, + should_build_docker = True, + docker_username = "foo", + docker_password = "bar", ) assert be.docker_tag == "dev" assert be.should_upload_docker assert not be.should_upload_pypi + assert be.should_upload_docker def test_buildenviron_rleasetag(): + # Simulates a tagged release on a release branch. be = cibuild.BuildEnviron( system = "Linux", root_dir = "/foo", travis_tag = "v0.0.1", travis_branch = "v0.x", + should_build_wheel = True, should_build_docker = True, should_build_pyinstaller = True, has_twine_creds = True, + docker_username = "foo", + docker_password = "bar", ) assert be.tag == "v0.0.1" assert be.branch == "v0.x" @@ -81,20 +101,31 @@ def test_buildenviron_rleasetag(): assert be.upload_dir == "0.0.1" assert be.docker_tag == "0.0.1" assert be.should_upload_pypi + assert be.should_upload_docker def test_buildenviron_branch(): + # Simulates a development branch on the main repo be = cibuild.BuildEnviron( system = "Linux", root_dir = "/foo", travis_tag = "", - travis_branch = "v0.x", + travis_branch = "mybranch", + + should_build_wheel = True, + should_build_docker = True, + should_build_pyinstaller = True, + has_twine_creds = True, + docker_username = "foo", + docker_password = "bar", ) assert be.tag == "" - assert be.branch == "v0.x" - assert be.version == "0.x" - assert be.upload_dir == "branches/0.x" + assert be.branch == "mybranch" + assert be.version == "mybranch" + assert be.upload_dir == "branches/mybranch" + assert not be.should_upload_pypi + assert not be.should_upload_docker def test_buildenviron_osx(tmpdir): From fdf0d9974e3216142c27d888d0eb7fdf1e832d73 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 24 May 2018 22:58:33 +1200 Subject: [PATCH 6/6] cibuild: more accurate PR test, better dump output --- release/cibuild.py | 29 +++++++++++++++++++---------- test/release/test_cibuild.py | 4 ++-- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/release/cibuild.py b/release/cibuild.py index 9d99d8d9c..373815275 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -210,16 +210,25 @@ class BuildEnviron: return False def dump_info(self, fp=sys.stdout): - print("BUILD PLATFORM_TAG=%s" % self.platform_tag, file=fp) - print("BUILD ROOT_DIR=%s" % self.root_dir, file=fp) - print("BUILD RELEASE_DIR=%s" % self.release_dir, file=fp) - print("BUILD BUILD_DIR=%s" % self.build_dir, file=fp) - print("BUILD DIST_DIR=%s" % self.dist_dir, file=fp) - print("BUILD BDISTS=%s" % self.bdists, file=fp) - print("BUILD TAG=%s" % self.tag, file=fp) - print("BUILD BRANCH=%s" % self.branch, file=fp) - print("BUILD VERSION=%s" % self.version, file=fp) - print("BUILD UPLOAD_DIR=%s" % self.upload_dir, file=fp) + lst = [ + "version", + "tag", + "branch", + "platform_tag", + "root_dir", + "release_dir", + "build_dir", + "dist_dir", + "bdists", + "upload_dir", + "should_build_wheel", + "should_build_pyinstaller", + "should_build_docker", + "should_upload_docker", + "should_upload_pypi", + ] + for attr in lst: + print("cibuild.%s=%s" % (attr, getattr(self, attr)), file=fp) def build_wheel(be: BuildEnviron): # pragma: no cover diff --git a/test/release/test_cibuild.py b/test/release/test_cibuild.py index 2b469d1be..71395c6d9 100644 --- a/test/release/test_cibuild.py +++ b/test/release/test_cibuild.py @@ -43,8 +43,8 @@ def test_buildenviron_pr(): # Simulates a PR. We build everything, but don't have access to secret # credential env variables. be = cibuild.BuildEnviron( - travis_tag = "v0.0.1", - travis_branch = "v0.x", + travis_tag = "", + travis_branch = "master", travis_pull_request = "true", should_build_wheel = True,