mitmproxy/release/cibuild.py

#!/usr/bin/env python3

import glob
import re
import contextlib
import os
import platform
import sys
import shutil
import subprocess
import tarfile
import zipfile
from os.path import join, abspath, dirname, exists, basename

import click
import cryptography.fernet


@contextlib.contextmanager
def chdir(path: str):
    old_dir = os.getcwd()
    os.chdir(path)
    yield
    os.chdir(old_dir)


class BuildError(Exception):
    pass


class BuildEnviron:
    PLATFORM_TAGS = {
        "Darwin": "osx",
        "Windows": "windows",
        "Linux": "linux",
    }

    def __init__(
        self,
        *,
        system = "",
        root_dir = "",

        travis_tag = "",
        travis_branch = "",
        travis_pull_request = "",

        appveyor_repo_tag_name = "",
        appveyor_repo_branch = "",
        appveyor_pull_request_number = "",

        build_wheel = "",
        build_docker = "",
        build_pyinstaller = "",

        has_aws_creds = False,
        has_twine_creds = False,

        docker_username = "",
        docker_password = "",
    ):
        self.system = system
        self.root_dir = root_dir

        self.travis_tag = travis_tag
        self.travis_branch = travis_branch
        self.travis_pull_request = travis_pull_request

        self.appveyor_repo_tag_name = appveyor_repo_tag_name
        self.appveyor_repo_branch = appveyor_repo_branch
        self.appveyor_pull_request_number = appveyor_pull_request_number

        self.has_aws_creds = has_aws_creds
        self.has_twine_creds = has_twine_creds
        self.docker_username = docker_username
        self.docker_password = docker_password

    @classmethod
    def from_env(klass):
        return klass(
            system = platform.system,
            root_dir = dirname(__file__),

            travis_tag = os.environ.get("TRAVIS_TAG", ""),
            travis_branch = os.environ.get("TRAVIS_BRANCH", ""),
            travis_pull_request = os.environ.get("TRAVIS_PULL_REQUEST"),

            appveyor_repo_tag_name = os.environ.get("APPVEYOR_REPO_TAG_NAME", ""),
            appveyor_repo_branch = os.environ.get("APPVEYOR_REPO_BRANCH", ""),
            appveyor_pull_request_number = os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"),

            build_wheel = "WHEEL" in os.environ,
            build_pyinstaller = "PYINSTALLER" in os.environ,
            build_docker = "DOCKER" in os.environ,

            has_aws_creds = "AWS_ACCESS_KEY_ID" in os.environ,
            has_twine_creds= (
                "TWINE_USERNAME" in os.environ and
                "TWINE_PASSWORD" in os.environ
            ),

            docker_username = os.environ.get("DOCKER_USERNAME"),
            docker_password = os.environ.get("DOCKER_PASSWORD"),
        )

    @property
    def has_docker_creds(self) -> bool:
        return self.docker_username and self.docker_password

    @property
    def is_pull_request(self) -> bool:
        if self.appveyor_pull_request_number:
            return True
        if self.travis_pull_request and self.travis_pull_request != "false":
            return True
        return False

    @property
    def tag(self):
        return self.travis_tag or self.appveyor_repo_tag_name

    @property
    def branch(self):
        return self.travis_branch or self.appveyor_repo_branch

    @property
    def version(self):
        name = self.tag or self.branch
        if not name:
            raise BuildError("Could not establish build name")
        return re.sub('^v', "", name)

    @property
    def upload_dir(self):
        if self.tag:
            return self.version
        else:
            return "branches/%s" % self.version

    @property
    def platform_tag(self):
        if self.system in self.PLATFORM_TAGS:
            return self.PLATFORM_TAGS[self.system]
        raise BuildError("Unsupported platform: %s" % self.system)

    @property
    def release_dir(self):
        return os.path.join(self.root_dir, "release")

    @property
    def build_dir(self):
        return os.path.join(self.release_dir, "build")

    @property
    def dist_dir(self):
        return os.path.join(self.release_dir, "dist")

    def archive(self, name):
        # ZipFile and tarfile have slightly different APIs. Fix that.
        if self.system == "Windows":
            a = zipfile.ZipFile(name, "w")
            a.add = a.write
            return a
        else:
            return tarfile.open(name, "w:gz")

    def archive_name(self, bdist: str) -> str:
        if self.system == "Windows":
            ext = "zip"
        else:
            ext = "tar.gz"
        return "{project}-{version}-{platform}.{ext}".format(
            project=bdist,
            version=self.version,
            platform=self.platform_tag,
            ext=ext
        )

    @property
    def bdists(self):
        ret = {
            "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
            "pathod": ["pathoc", "pathod"]
        }
        if self.system == "Windows":
            ret["mitmproxy"].remove("mitmproxy")
        return ret

    def dump_info(self, fp=sys.stdout):
        print("BUILD PLATFORM_TAG=%s" % self.platform_tag, file=fp)
        print("BUILD ROOT_DIR=%s" % self.root_dir, file=fp)
        print("BUILD RELEASE_DIR=%s" % self.release_dir, file=fp)
        print("BUILD BUILD_DIR=%s" % self.build_dir, file=fp)
        print("BUILD DIST_DIR=%s" % self.dist_dir, file=fp)
        print("BUILD BDISTS=%s" % self.bdists, file=fp)
        print("BUILD TAG=%s" % self.tag, file=fp)
        print("BUILD BRANCH=%s" % self.branch, file=fp)
        print("BUILD VERSION=%s" % self.version, file=fp)
        print("BUILD UPLOAD_DIR=%s" % self.upload_dir, file=fp)


def build_wheel(be: BuildEnviron):
    click.echo("Building wheel...")
    subprocess.check_call([
        "python",
        "setup.py",
        "-q",
        "bdist_wheel",
        "--dist-dir", be.dist_dir,
    ])
    whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
    click.echo("Found wheel package: {}".format(whl))
    subprocess.check_call(["tox", "-e", "wheeltest", "--", whl])
    return whl


def build_docker_image(be: BuildEnviron, whl: str):
    click.echo("Building Docker image...")
    subprocess.check_call([
        "docker",
        "build",
        "--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, be.root_dir)),
        "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(basename(whl)),
        "--file", "docker/Dockerfile",
        "."
    ])


def build_pyinstaller(be: BuildEnviron):
    click.echo("Building pyinstaller package...")

    PYINSTALLER_SPEC = join(be.release_dir, "specs")
    # PyInstaller 3.2 does not bundle pydivert's Windivert binaries
    PYINSTALLER_HOOKS = join(be.release_dir, "hooks")
    PYINSTALLER_TEMP = join(be.build_dir, "pyinstaller")
    PYINSTALLER_DIST = join(be.build_dir, "binaries", be.platform_tag)

    # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
    # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
    if platform.system() == "Windows":
        PYINSTALLER_ARGS = [
            # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
            "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
        ]
    else:
        PYINSTALLER_ARGS = []

    if exists(PYINSTALLER_TEMP):
        shutil.rmtree(PYINSTALLER_TEMP)
    if exists(PYINSTALLER_DIST):
        shutil.rmtree(PYINSTALLER_DIST)

    for bdist, tools in sorted(be.bdists.items()):
        with be.archive(join(be.dist_dir, be.archive_name(bdist))) as archive:
            for tool in tools:
                # We can't have a folder and a file with the same name.
                if tool == "mitmproxy":
                    tool = "mitmproxy_main"
                # This is PyInstaller, so it messes up paths.
                # We need to make sure that we are in the spec folder.
                with chdir(PYINSTALLER_SPEC):
                    click.echo("Building PyInstaller %s binary..." % tool)
                    excludes = []
                    if tool != "mitmweb":
                        excludes.append("mitmproxy.tools.web")
                    if tool != "mitmproxy_main":
                        excludes.append("mitmproxy.tools.console")

                    subprocess.check_call(
                        [
                            "pyinstaller",
                            "--clean",
                            "--workpath", PYINSTALLER_TEMP,
                            "--distpath", PYINSTALLER_DIST,
                            "--additional-hooks-dir", PYINSTALLER_HOOKS,
                            "--onefile",
                            "--console",
                            "--icon", "icon.ico",
                            # This is PyInstaller, so setting a
                            # different log level obviously breaks it :-)
                            # "--log-level", "WARN",
                        ]
                        + [x for e in excludes for x in ["--exclude-module", e]]
                        + PYINSTALLER_ARGS
                        + [tool]
                    )
                    # Delete the spec file - we're good without.
                    os.remove("{}.spec".format(tool))

                # Test if it works at all O:-)
                executable = join(PYINSTALLER_DIST, tool)
                if platform.system() == "Windows":
                    executable += ".exe"

                # Remove _main suffix from mitmproxy executable
                if "_main" in executable:
                    shutil.move(
                        executable,
                        executable.replace("_main", "")
                    )
                    executable = executable.replace("_main", "")

                click.echo("> %s --version" % executable)
                click.echo(subprocess.check_output([executable, "--version"]).decode())

                archive.add(executable, basename(executable))
        click.echo("Packed {}.".format(be.archive_name(bdist)))


@click.group(chain=True)
def cli():
    """
    mitmproxy build tool
    """
    pass


@cli.command("build")
def build():
    """
        Build a binary distribution
    """
    be = BuildEnviron.from_env()
    be.dump_info()

    os.makedirs(be.dist_dir, exist_ok=True)

    if be.build_wheel:
        whl = build_wheel(be)
        # Docker image requires wheels
        if be.build_docker:
            build_docker_image(whl)
    if be.build_pyinstaller:
        build_pyinstaller()


@cli.command("upload")
def upload():
    """
        Upload build artifacts

        Uploads the wheels package to PyPi.
        Uploads the Pyinstaller and wheels packages to the snapshot server.
        Pushes the Docker image to Docker Hub.
    """
    be = BuildEnviron.from_env()

    if be.is_pull_request:
        click.echo("Refusing to upload artifacts from a pull request!")
        return

    if be.has_aws_creds:
        subprocess.check_call([
            "aws", "s3", "cp",
            "--acl", "public-read",
            be.dist_dir + "/",
            "s3://snapshots.mitmproxy.org/{}/".format(be.upload_dir),
            "--recursive",
        ])

    upload_pypi = (be.tag and be.build_wheel and be.has_twine_creds)
    if upload_pypi:
        whl = glob.glob(join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
        click.echo("Uploading {} to PyPi...".format(whl))
        subprocess.check_call(["twine", "upload", whl])

    upload_docker = (
        (be.tag or be.branch == "master") and
        be.build_docker,
        be.has_docker_creds,
    )
    if upload_docker:
        docker_tag = "dev" if be.branch == "master" else be.version

        click.echo("Uploading Docker image to tag={}...".format(docker_tag))
        subprocess.check_call([
            "docker",
            "login",
            "-u", be.docker_username,
            "-p", be.docker_password,
        ])
        subprocess.check_call([
            "docker",
            "push",
            "mitmproxy/mitmproxy:{}".format(docker_tag),
        ])


@cli.command("decrypt")
@click.argument('infile', type=click.File('rb'))
@click.argument('outfile', type=click.File('wb'))
@click.argument('key', envvar='RTOOL_KEY')
def decrypt(infile, outfile, key):
    f = cryptography.fernet.Fernet(key.encode())
    outfile.write(f.decrypt(infile.read()))


if __name__ == "__main__":
    cli()