mitmproxy/release/cibuild.py

#!/usr/bin/env python3

import glob
import re
import contextlib
import os
import platform
import sys
import shutil
import subprocess
import tarfile
import zipfile
from os.path import join, abspath, dirname, exists, basename

import click
import cryptography.fernet

# ZipFile and tarfile have slightly different APIs. Fix that.
if platform.system() == "Windows":
    def Archive(name):
        a = zipfile.ZipFile(name, "w")
        a.add = a.write
        return a
else:
    def Archive(name):
        return tarfile.open(name, "w:gz")

PLATFORM_TAG = {
    "Darwin": "osx",
    "Windows": "windows",
    "Linux": "linux",
}.get(platform.system(), platform.system())

ROOT_DIR = abspath(join(dirname(__file__), ".."))
RELEASE_DIR = join(ROOT_DIR, "release")
BUILD_DIR = join(RELEASE_DIR, "build")
DIST_DIR = join(RELEASE_DIR, "dist")

BDISTS = {
    "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
    "pathod": ["pathoc", "pathod"]
}
if platform.system() == "Windows":
    BDISTS["mitmproxy"].remove("mitmproxy")

TOOLS = [
    tool
    for tools in sorted(BDISTS.values())
    for tool in tools
]

TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
if TAG:
    VERSION = re.sub('^v', '', TAG)
    UPLOAD_DIR = VERSION
elif BRANCH:
    VERSION = re.sub('^v', '', BRANCH)
    UPLOAD_DIR = "branches/%s" % VERSION
else:
    print("Could not establish build name - exiting." % BRANCH)
    sys.exit(0)

print("BUILD PLATFORM_TAG=%s" % PLATFORM_TAG)
print("BUILD ROOT_DIR=%s" % ROOT_DIR)
print("BUILD RELEASE_DIR=%s" % RELEASE_DIR)
print("BUILD BUILD_DIR=%s" % BUILD_DIR)
print("BUILD DIST_DIR=%s" % DIST_DIR)
print("BUILD BDISTS=%s" % BDISTS)
print("BUILD TAG=%s" % TAG)
print("BUILD BRANCH=%s" % BRANCH)
print("BUILD VERSION=%s" % VERSION)
print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR)


def archive_name(bdist: str) -> str:
    if platform.system() == "Windows":
        ext = "zip"
    else:
        ext = "tar.gz"
    return "{project}-{version}-{platform}.{ext}".format(
        project=bdist,
        version=VERSION,
        platform=PLATFORM_TAG,
        ext=ext
    )


@contextlib.contextmanager
def chdir(path: str):
    old_dir = os.getcwd()
    os.chdir(path)
    yield
    os.chdir(old_dir)


@click.group(chain=True)
def cli():
    """
    mitmproxy build tool
    """
    pass


@cli.command("build")
def build():
    """
    Build a binary distribution
    """
    os.makedirs(DIST_DIR, exist_ok=True)

    if "WHEEL" in os.environ:
        whl = build_wheel()
    else:
        click.echo("Not building wheels.")

    if "WHEEL" in os.environ and "DOCKER" in os.environ:
        # Docker image requires wheels
        build_docker_image(whl)
    else:
        click.echo("Not building Docker image.")

    if "PYINSTALLER" in os.environ:
        build_pyinstaller()
    else:
        click.echo("Not building PyInstaller packages.")


def build_wheel():
    click.echo("Building wheel...")
    subprocess.check_call([
        "python",
        "setup.py",
        "-q",
        "bdist_wheel",
        "--dist-dir", DIST_DIR,
    ])

    whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
    click.echo("Found wheel package: {}".format(whl))

    subprocess.check_call([
        "tox",
        "-e", "wheeltest",
        "--",
        whl
    ])

    return whl


def build_docker_image(whl):
    click.echo("Building Docker image...")
    subprocess.check_call([
        "docker",
        "build",
        "--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, ROOT_DIR)),
        "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(basename(whl)),
        "--file", "docker/Dockerfile",
        "."
    ])


def build_pyinstaller():
    PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
    # PyInstaller 3.2 does not bundle pydivert's Windivert binaries
    PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
    PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
    PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)

    # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
    # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
    if platform.system() == "Windows":
        PYINSTALLER_ARGS = [
            # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
            "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
        ]
    else:
        PYINSTALLER_ARGS = []

    if exists(PYINSTALLER_TEMP):
        shutil.rmtree(PYINSTALLER_TEMP)
    if exists(PYINSTALLER_DIST):
        shutil.rmtree(PYINSTALLER_DIST)

    for bdist, tools in sorted(BDISTS.items()):
        with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
            for tool in tools:
                # We can't have a folder and a file with the same name.
                if tool == "mitmproxy":
                    tool = "mitmproxy_main"
                # This is PyInstaller, so it messes up paths.
                # We need to make sure that we are in the spec folder.
                with chdir(PYINSTALLER_SPEC):
                    click.echo("Building PyInstaller %s binary..." % tool)
                    excludes = []
                    if tool != "mitmweb":
                        excludes.append("mitmproxy.tools.web")
                    if tool != "mitmproxy_main":
                        excludes.append("mitmproxy.tools.console")

                    subprocess.check_call(
                        [
                            "pyinstaller",
                            "--clean",
                            "--workpath", PYINSTALLER_TEMP,
                            "--distpath", PYINSTALLER_DIST,
                            "--additional-hooks-dir", PYINSTALLER_HOOKS,
                            "--onefile",
                            "--console",
                            "--icon", "icon.ico",
                            # This is PyInstaller, so setting a
                            # different log level obviously breaks it :-)
                            # "--log-level", "WARN",
                        ]
                        + [x for e in excludes for x in ["--exclude-module", e]]
                        + PYINSTALLER_ARGS
                        + [tool]
                    )
                    # Delete the spec file - we're good without.
                    os.remove("{}.spec".format(tool))

                # Test if it works at all O:-)
                executable = join(PYINSTALLER_DIST, tool)
                if platform.system() == "Windows":
                    executable += ".exe"

                # Remove _main suffix from mitmproxy executable
                if "_main" in executable:
                    shutil.move(
                        executable,
                        executable.replace("_main", "")
                    )
                    executable = executable.replace("_main", "")

                click.echo("> %s --version" % executable)
                click.echo(subprocess.check_output([executable, "--version"]).decode())

                archive.add(executable, basename(executable))
        click.echo("Packed {}.".format(archive_name(bdist)))


@cli.command("upload")
def upload():
    """
        Upload build artifacts

        Uploads the wheels package to PyPi.
        Uploads the Pyinstaller and wheels packages to the snapshot server.
        Pushes the Docker image to Docker Hub.
    """

    # Our credentials are only available from within the main repository and not forks.
    # We need to prevent uploads from all BUT the branches in the main repository.
    # Pull requests and master-branches of forks are not allowed to upload.
    is_pull_request = (
        ("TRAVIS_PULL_REQUEST" in os.environ and os.environ["TRAVIS_PULL_REQUEST"] != "false") or
        "APPVEYOR_PULL_REQUEST_NUMBER" in os.environ
    )
    if is_pull_request:
        click.echo("Refusing to upload artifacts from a pull request!")
        return

    if "AWS_ACCESS_KEY_ID" in os.environ:
        subprocess.check_call([
            "aws", "s3", "cp",
            "--acl", "public-read",
            DIST_DIR + "/",
            "s3://snapshots.mitmproxy.org/{}/".format(UPLOAD_DIR),
            "--recursive",
        ])

    upload_pypi = (
        TAG and
        "WHEEL" in os.environ and
        "TWINE_USERNAME" in os.environ and
        "TWINE_PASSWORD" in os.environ
    )
    if upload_pypi:
        whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
        click.echo("Uploading {} to PyPi...".format(whl))
        subprocess.check_call([
            "twine",
            "upload",
            whl
        ])

    upload_docker = (
        (TAG or BRANCH == "master") and
        "DOCKER" in os.environ and
        "DOCKER_USERNAME" in os.environ and
        "DOCKER_PASSWORD" in os.environ
    )
    if upload_docker:
        docker_tag = "dev" if BRANCH == "master" else VERSION

        click.echo("Uploading Docker image to tag={}...".format(docker_tag))
        subprocess.check_call([
            "docker",
            "login",
            "-u", os.environ["DOCKER_USERNAME"],
            "-p", os.environ["DOCKER_PASSWORD"],
        ])
        subprocess.check_call([
            "docker",
            "push",
            "mitmproxy/mitmproxy:{}".format(docker_tag),
        ])


@cli.command("decrypt")
@click.argument('infile', type=click.File('rb'))
@click.argument('outfile', type=click.File('wb'))
@click.argument('key', envvar='RTOOL_KEY')
def decrypt(infile, outfile, key):
    f = cryptography.fernet.Fernet(key.encode())
    outfile.write(f.decrypt(infile.read()))


if __name__ == "__main__":
    cli()