mitmproxy/build

#!/usr/bin/env python
from __future__ import (
    absolute_import, print_function, division, unicode_literals
)
from contextlib import contextmanager
from os.path import dirname, realpath, join, exists, normpath
import os
import shutil
import subprocess
import glob
import re
import shlex
import click

# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
if os.name == "nt":
    VENV_BIN = "Scripts"
else:
    VENV_BIN = "bin"

RELEASE_DIR = join(dirname(realpath(__file__)))
ROOT_DIR = join(RELEASE_DIR, "..")
MITMPROXY_DIR = join(ROOT_DIR, "mitmproxy")
DIST_DIR = join(MITMPROXY_DIR, "dist")
TEST_VENV_DIR = join(RELEASE_DIR, "venv")

PROJECTS = ("netlib", "pathod", "mitmproxy")
TOOLS = {
    "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
    "pathod": ["pathod", "pathoc"],
    "netlib": []
}
if os.name == "nt":
    TOOLS["mitmproxy"].remove("mitmproxy")
VERSION_FILES = {
    "mitmproxy": normpath(join(ROOT_DIR, "mitmproxy/libmproxy/version.py")),
    "pathod": normpath(join(ROOT_DIR, "pathod/libpathod/version.py")),
    "netlib": normpath(join(ROOT_DIR, "netlib/netlib/version.py")),
}


@contextmanager
def empty_pythonpath():
    """
    Make sure that the regular python installation is not on the python path,
    which would give us access to modules installed outside of our virtualenv.
    """
    pythonpath = os.environ["PYTHONPATH"]
    os.environ["PYTHONPATH"] = ""
    yield
    os.environ["PYTHONPATH"] = pythonpath


@contextmanager
def chdir(path):
    old_dir = os.getcwd()
    os.chdir(path)
    yield
    os.chdir(old_dir)


@click.group(chain=True)
def cli():
    """
    mitmproxy build tool
    """
    pass


@cli.command("contributors")
@click.option(
    '--project', '-p', 'projects',
    multiple=True, type=click.Choice(PROJECTS), default=PROJECTS
)
def contributors(projects):
    """
        Update CONTRIBUTORS.md
    """
    for project in PROJECTS:
        if project not in projects:
            continue
        with chdir(os.path.join(ROOT_DIR, project)):
            print("Updating %s/CONTRIBUTORS..."%project)
            contributors_data = subprocess.check_output(
                shlex.split("git shortlog -n -s")
            )
            with open("CONTRIBUTORS", "w+") as f:
                f.write(contributors_data)


@cli.command("docs")
def docs():
    """
    Render the docs
    """
    print("Rendering the docs...")
    subprocess.check_call([
        "cshape",
        join(MITMPROXY_DIR, "doc-src"),
        join(MITMPROXY_DIR, "doc")
    ])


@cli.command("set-version")
@click.option(
    '--project', '-p', 'projects',
    multiple=True, type=click.Choice(PROJECTS), default=PROJECTS
)
@click.argument('version')
def set_version(projects, version):
    """
    Update version information
    """
    print("Update versions...")
    version = ", ".join(version.split("."))
    for project, version_file in VERSION_FILES.items():
        if project not in projects:
            continue
        print("Update %s..." % version_file)
        with open(version_file, "rb") as f:
            content = f.read()
        new_content = re.sub(
            r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version,
            content
        )
        with open(version_file, "wb") as f:
            f.write(new_content)


@cli.command("git")
@click.option(
    '--project', '-p', 'projects',
    multiple=True, type=click.Choice(PROJECTS), default=PROJECTS
)
@click.argument('args', nargs=-1, required=True)
def git(projects, args):
    """
    Run a git command on every project
    """
    args = ["git"] + list(args)
    for project in projects:
        print("%s> %s..." % (project, " ".join(args)))
        subprocess.check_call(
            args,
            cwd=join(ROOT_DIR, project)
        )


@cli.command("sdist")
@click.option(
    '--project', '-p', 'projects',
    multiple=True, type=click.Choice(PROJECTS), default=PROJECTS
)
def sdist(projects):
    """
    Build a source distribution
    """
    with empty_pythonpath():
        print("Building release...")
        if exists(DIST_DIR):
            shutil.rmtree(DIST_DIR)
        for project in projects:
            print("Creating %s source distribution..." % project)
            subprocess.check_call(
                [
                    "python", "./setup.py",
                    "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar"
                ],
                cwd=join(ROOT_DIR, project)
            )


@cli.command("test")
@click.option(
    '--project', '-p', 'projects',
    multiple=True, type=click.Choice(PROJECTS), default=PROJECTS
)
@click.pass_context
def test(ctx, projects):
    """
    Test the source distribution
    """
    if not exists(DIST_DIR):
        ctx.invoke(sdist)

    with empty_pythonpath():
        print("Creating virtualenv for test install...")
        if exists(TEST_VENV_DIR):
            shutil.rmtree(TEST_VENV_DIR)
        subprocess.check_call(["virtualenv", "-q", TEST_VENV_DIR])

        pip = join(TEST_VENV_DIR, VENV_BIN, "pip")
        with chdir(DIST_DIR):
            for project in projects:
                print("Installing %s..." % project)
                dist = join(ROOT_DIR, project)
                subprocess.check_call([pip, "install", "-q", dist])

            print("Running binaries...")
            for project in projects:
                for tool in TOOLS[project]:
                    tool = join(TEST_VENV_DIR, VENV_BIN, tool)
                    print(tool)
                    print(subprocess.check_output([tool, "--version"]))

            print("Virtualenv available for further testing:")
            print(
                "source %s" % normpath(
                    join(TEST_VENV_DIR, VENV_BIN, "activate")
                )
            )


@cli.command("upload")
@click.option('--username', prompt=True)
@click.password_option(confirmation_prompt=False)
@click.option('--repository', default="pypi")
def upload_release(username, password, repository):
    """
    Upload source distributions to PyPI
    """
    print("Uploading distributions...")
    subprocess.check_call([
        "twine",
        "upload",
        "-u", username,
        "-p", password,
        "-r", repository,
        "%s/*" % DIST_DIR
    ])


# TODO: Fully automate build process.
# This wizard is missing OSX builds and updating mitmproxy.org.
@cli.command("wizard")
@click.option('--version', prompt=True)
@click.option('--username', prompt="PyPI Username")
@click.password_option(confirmation_prompt=False, prompt="PyPI Password")
@click.option('--repository', default="pypi")
@click.option(
    '--project', '-p', 'projects',
    multiple=True, type=click.Choice(PROJECTS), default=PROJECTS
)
@click.pass_context
def wizard(ctx, version, username, password, repository, projects):
    """
    Interactive Release Wizard
    """
    for project in projects:
        if subprocess.check_output(
            ["git", "status", "--porcelain"], cwd=join(ROOT_DIR, project)
        ):
            raise RuntimeError("%s repository is not clean." % project)

    # Build test release
    ctx.invoke(sdist, projects=projects)
    ctx.invoke(test, projects=projects)
    click.confirm("Please test the release now. Is it ok?", abort=True)

    # bump version, update docs and contributors
    ctx.invoke(set_version, version=version, projects=projects)
    ctx.invoke(docs)
    ctx.invoke(contributors)

    # version bump commit + tag
    ctx.invoke(
        git, args=["commit", "-a", "-m", "bump version"], projects=projects
    )
    ctx.invoke(git, args=["tag", "v" + version], projects=projects)
    ctx.invoke(git, args=["push"], projects=projects)
    ctx.invoke(git, args=["push", "--tags"], projects=projects)

    # Re-invoke sdist with bumped version
    ctx.invoke(sdist, projects=projects)
    click.confirm("All good, can upload to PyPI?", abort=True)
    ctx.invoke(
        upload_release,
        username=username, password=password, repository=repository
    )
    click.echo("All done!")


if __name__ == "__main__":
    cli()