streamline build process

release/build.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+
+from os.path import dirname, realpath, join
+from os import chdir, mkdir, getcwd
+import os
+import shutil
+import subprocess
+import tempfile
+import glob
+from shlex import split
+from contextlib import contextmanager
+import click
+
+# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
+# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
+if os.name == "nt":
+    venv_bin = "Scripts"
+else:
+    venv_bin = "bin"
+
+_root_dir = join(dirname(realpath(__file__)), "..", "..")
+projects = ("netlib", "pathod", "mitmproxy")
+dirs = {x: join(_root_dir, x) for x in projects}
+dirs["root"] = _root_dir
+dirs["dist"] = join(dirs["mitmproxy"], "dist")
+
+tools = ["mitmweb", "mitmdump", "pathod", "pathoc"]
+if os.name != "nt":
+    tools.append("mitmproxy")
+
+
+# Python 3: replace with tempfile.TemporaryDirectory
+@contextmanager
+def tmpdir(*args, **kwargs):
+    orig_workdir = os.getcwd()
+    temp_workdir = tempfile.mkdtemp(*args, **kwargs)
+    yield temp_workdir
+    os.chdir(orig_workdir)
+    shutil.rmtree(temp_workdir)
+
+
+@click.group(chain=True)
+def cli():
+    """
+    mitmproxy build tool
+    """
+    pass
+
+
+@cli.command("contributors")
+def update_contributors():
+    print("Updating CONTRIBUTORS.md...")
+    contributors = subprocess.check_output(split("git shortlog -n -s"))
+    with open(join(dirs["mitmproxy"], "CONTRIBUTORS"), "w") as f:
+        f.write(contributors)
+
+
+@cli.command("docs")
+def render_docs():
+    print("Rendering the docs...")
+    subprocess.check_call([
+        "cshape",
+        join(dirs["mitmproxy"], "doc-src"),
+        join(dirs["mitmproxy"], "doc")
+    ])
+
+
+@cli.command("release")
+def build_release():
+    # Make sure that the regular python installation is not on the python path!
+    os.environ["PYTHONPATH"] = ""
+
+    with tmpdir("mitmproxy_release") as tmp:
+
+        print("Building release...")
+        print("Temp directory: %s" % tmp)
+
+        for project in projects:
+            print("Creating %s source distribution..." % project)
+            print(dirs[project])
+            subprocess.check_call(["python", "./setup.py", "-q", "sdist", "--dist-dir", tmp, "--formats=gztar"], cwd=dirs[project])
+
+        print("Creating virtualenv for test install...")
+        venv_dir = join(tmp, "venv")
+        subprocess.check_call(["virtualenv", "-q", venv_dir])
+
+        pip = join(venv_dir, venv_bin, "pip")
+        chdir(tmp)
+        for project in projects:
+            print("Installing %s..." % project)
+            dist = glob.glob("./%s*" % project)[0]
+            subprocess.check_call([pip, "install", "-q", dist])
+
+        print("Running binaries...")
+        for tool in tools:
+            tool = join(venv_dir, venv_bin, tool)
+            print(tool)
+            print(subprocess.check_output([tool, "--version"]))
+
+        shutil.rmtree(venv_dir)
+        shutil.rmtree(dirs["dist"])
+        shutil.copytree(tmp, dirs["dist"])
+
+        # TODO: Use the output of this step for further processing, i.e. test and then uplaod using twine
+        # https://packaging.python.org/en/latest/distributing.html#upload-your-distributions
+        # https://github.com/pypa/twine/issues/116
+
+
+@cli.command("upload")
+@click.option('--username', prompt=True)
+@click.password_option(confirmation_prompt=False)
+@click.option('--repository', default="pypi")
+def upload_release(username, password, repository):
+    print("Uploading distributions...")
+    subprocess.check_call([
+        "twine",
+        "upload",
+        "-u", username,
+        "-p", password,
+        "-r", repository,
+        "%s/*" % dirs["dist"]
+    ])
+
+if __name__ == "__main__":
+    cli()

release/contributors

@@ -1,2 +0,0 @@
-#!/bin/sh
-git shortlog -n -s

release/release-checklist

@@ -8,14 +8,14 @@
 - Ensure that the website style assets have been compiled for production, and
 synced to the docs.
 
-- Render the docs:
-    cshape doc-src doc
+- Render the docs, update CONTRIBUTORS file:
+    ./release/build.py docs contributors
 
 - Run the test release, make sure the output is sensible
-    ./release/test-release
+    ./release/build.py release
 
 - Build the OSX binaries
-    - Follow instructions in osxbinaries
+    - Follow instructions in osx-binaries
     - Move to download dir:
         mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download
 
@@ -28,10 +28,12 @@ synced to the docs.
 
 - Upload to pypi for each project:
 
-    python ./setup.py sdist upload
+    ./release/build.py upload
 
 - Now bump the version number to be ready for the next cycle:
 
+    TODO: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now?
+
     mitmproxy/libmproxy/version.py
     netlib/netlib/version.py
     pathod/libpathod/version.py

setup.py

@@ -38,6 +38,7 @@ dev_deps = {
     "nose>=1.3.0",
     "nose-cov>=1.6",
     "coveralls>=0.4.1",
+    "click>=4.1",
     "pathod>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION),
     "countershape"
 }