From 2bbfcfae927c89d42b0b0c3d1abac268ee68dff7 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 17 May 2018 11:25:32 +0200 Subject: [PATCH] improve release workflow --- release/README.md | 28 ++-- release/ci.py | 114 +++++++++------- release/rtool.py | 328 ++++++++++++++++++++++++++++++++++++++++++---- tox.ini | 14 +- 4 files changed, 401 insertions(+), 83 deletions(-) diff --git a/release/README.md b/release/README.md index d15f88a01..044e0dc9e 100644 --- a/release/README.md +++ b/release/README.md @@ -1,12 +1,12 @@ # Release Checklist -Make sure run all these steps on the correct branch you want to create a new +Make sure to run all these steps on the correct branch you want to create a new release for! The command examples assume that you have a git remote called `upstream` that points to the `mitmproxy/mitmproxy` repo. - Verify that `mitmproxy/version.py` is correct - Update CHANGELOG -- Update CONTRIBUTORS +- Update CONTRIBUTORS: `git shortlog -n -s > CONTRIBUTORS` - Verify that all CI tests pass - Create a major version branch - e.g. `v4.x`. Assuming you have a remote repo called `upstream` that points to the mitmproxy/mitmproxy repo:: - `git checkout -b v4.x upstream/master` @@ -20,16 +20,20 @@ release for! The command examples assume that you have a git remote called - Wait for tag CI to complete ## GitHub Release -- Create release notice on Github [here](https://github.com/mitmproxy/mitmproxy/releases/new) if not already auto-created by the tag. -- We DO NOT upload release artifacts to GitHub anymore. Simply add the following snippet to the notice: +- Create release notice on Github + [here](https://github.com/mitmproxy/mitmproxy/releases/new) if not already + auto-created by the tag. +- We DO NOT upload release artifacts to GitHub anymore. Simply add the + following snippet to the notice: `You can find the latest release packages on our snapshot server: https://snapshots.mitmproxy.org/v` ## PyPi -- Upload the whl file you downloaded in the prevous step -- `twine upload ./tmp/snap/mitmproxy-4.0.0-py3-none-any.whl` +- The created wheel is uploaded to PyPi automatically +- Please check https://pypi.python.org/pypi/mitmproxy about the latest version ## Homebrew -- The Homebrew maintainers are typically very fast and detect our new relese within a day. +- The Homebrew maintainers are typically very fast and detect our new relese + within a day. - If you feel the need, you can run this from a macOS machine: `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/v` @@ -52,9 +56,10 @@ release for! The command examples assume that you have a git remote called - Check the build details page again ## Website - - Update version here: https://github.com/mitmproxy/www/blob/master/src/config.toml - - `./build && ./upload-test` - - If everything looks alright: `./upload-prod` + - Update version here: + https://github.com/mitmproxy/www/blob/master/src/config.toml + - Run `./build && ./upload-test` + - If everything looks alright, run `./upload-prod` ## Docs - Make sure you've uploaded the previous version's docs to archive @@ -64,4 +69,5 @@ release for! The command examples assume that you have a git remote called ## Prepare for next release - - Last but not least, bump the version on master in [https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) for major releases. + - Last but not least, bump the version on master in + [https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) for major releases. diff --git a/release/ci.py b/release/ci.py index 94b1f13d5..7550aae42 100755 --- a/release/ci.py +++ b/release/ci.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import glob +import re import contextlib import os import platform @@ -73,17 +75,17 @@ TOOLS = [ TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None)) BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None)) if TAG: - VERSION = TAG + VERSION = re.sub('^v', '', TAG) UPLOAD_DIR = VERSION elif BRANCH: - VERSION = BRANCH + VERSION = re.sub('^v', '', BRANCH) UPLOAD_DIR = "branches/%s" % VERSION else: print("Could not establish build name - exiting." % BRANCH) sys.exit(0) - print("BUILD VERSION=%s" % VERSION) +print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR) def archive_name(bdist: str) -> str: @@ -99,23 +101,6 @@ def archive_name(bdist: str) -> str: ) -def wheel_name() -> str: - return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION) - - -def installer_name() -> str: - ext = { - "Windows": "exe", - "Darwin": "dmg", - "Linux": "run" - }[platform.system()] - return "mitmproxy-{version}-{platform}-installer.{ext}".format( - version=VERSION, - platform=PLATFORM_TAG, - ext=ext, - ) - - @contextlib.contextmanager def chdir(path: str): old_dir = os.getcwd() @@ -134,7 +119,7 @@ def cli(): @cli.command("info") def info(): - print("Version: %s" % VERSION) + click.echo("Version: %s" % VERSION) @cli.command("build") @@ -142,23 +127,41 @@ def build(): """ Build a binary distribution """ + os.makedirs(DIST_DIR, exist_ok=True) + if "WHEEL" in os.environ: + build_wheel() + else: + click.echo("Not building wheels.") + build_pyinstaller() + + +def build_wheel(): + click.echo("Building wheel...") + subprocess.check_call([ + "python", + "setup.py", + "-q", + "bdist_wheel", + "--dist-dir", DIST_DIR, + ]) + + whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0] + click.echo("Found wheel package: {}".format(whl)) + + subprocess.check_call([ + "tox", + "-e", "wheeltest", + "--", + whl + ]) + + +def build_pyinstaller(): if exists(PYINSTALLER_TEMP): shutil.rmtree(PYINSTALLER_TEMP) if exists(PYINSTALLER_DIST): shutil.rmtree(PYINSTALLER_DIST) - os.makedirs(DIST_DIR, exist_ok=True) - - if "WHEEL" in os.environ: - print("Building wheel...") - subprocess.check_call( - [ - "python", - "setup.py", "-q", "bdist_wheel", - "--dist-dir", "release/dist", - ] - ) - for bdist, tools in sorted(BDISTS.items()): with Archive(join(DIST_DIR, archive_name(bdist))) as archive: for tool in tools: @@ -168,7 +171,7 @@ def build(): # This is PyInstaller, so it messes up paths. # We need to make sure that we are in the spec folder. with chdir(PYINSTALLER_SPEC): - print("Building %s binary..." % tool) + click.echo("Building %s binary..." % tool) excludes = [] if tool != "mitmweb": excludes.append("mitmproxy.tools.web") @@ -209,11 +212,11 @@ def build(): ) executable = executable.replace("_main", "") - print("> %s --version" % executable) - print(subprocess.check_output([executable, "--version"]).decode()) + click.echo("> %s --version" % executable) + click.echo(subprocess.check_output([executable, "--version"]).decode()) archive.add(executable, basename(executable)) - print("Packed {}.".format(archive_name(bdist))) + click.echo("Packed {}.".format(archive_name(bdist))) def is_pr(): @@ -229,25 +232,40 @@ def is_pr(): @cli.command("upload") def upload(): """ - Upload snapshot to snapshot server + Upload build artifacts to snapshot server and + upload wheel package to PyPi """ # This requires some explanation. The AWS access keys are only exposed to # privileged builds - that is, they are not available to PRs from forks. # However, they ARE exposed to PRs from a branch within the main repo. This # check catches that corner case, and prevents an inadvertent upload. if is_pr(): - print("Refusing to upload a pull request") + click.echo("Refusing to upload a pull request") return + if "AWS_ACCESS_KEY_ID" in os.environ: - subprocess.check_call( - [ - "aws", "s3", "cp", - "--acl", "public-read", - DIST_DIR + "/", - "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR, - "--recursive", - ] - ) + subprocess.check_call([ + "aws", "s3", "cp", + "--acl", "public-read", + DIST_DIR + "/", + "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR, + "--recursive", + ]) + + upload_pypi = ( + TAG and + "WHEEL" in os.environ and + "TWINE_USERNAME" in os.environ and + "TWINE_PASSWORD" in os.environ + ) + if upload_pypi: + filename = "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION) + click.echo("Uploading {} to PyPi...".format(filename)) + subprocess.check_call([ + "twine", + "upload", + join(DIST_DIR, filename) + ]) @cli.command("decrypt") diff --git a/release/rtool.py b/release/rtool.py index 68ff02ab1..14a0a0784 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -1,22 +1,78 @@ #!/usr/bin/env python3 import contextlib +import fnmatch import os -import sys import platform +import re import runpy import shlex +import shutil import subprocess -from os.path import join, abspath, dirname +import tarfile +import zipfile +from os.path import join, abspath, dirname, exists, basename -import cryptography.fernet import click +import cryptography.fernet +import pysftp +# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes +# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ +if platform.system() == "Windows": + VENV_BIN = "Scripts" + PYINSTALLER_ARGS = [ + # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly. + "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86", + ] +else: + VENV_BIN = "bin" + PYINSTALLER_ARGS = [] + +# ZipFile and tarfile have slightly different APIs. Fix that. +if platform.system() == "Windows": + def Archive(name): + a = zipfile.ZipFile(name, "w") + a.add = a.write + return a +else: + def Archive(name): + return tarfile.open(name, "w:gz") + +PLATFORM_TAG = { + "Darwin": "osx", + "Windows": "windows", + "Linux": "linux", +}.get(platform.system(), platform.system()) ROOT_DIR = abspath(join(dirname(__file__), "..")) RELEASE_DIR = join(ROOT_DIR, "release") + +BUILD_DIR = join(RELEASE_DIR, "build") DIST_DIR = join(RELEASE_DIR, "dist") + +PYINSTALLER_SPEC = join(RELEASE_DIR, "specs") +# PyInstaller 3.2 does not bundle pydivert's Windivert binaries +PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks") +PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") +PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG) + +VENV_DIR = join(BUILD_DIR, "venv") + +# Project Configuration VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py") +BDISTS = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathoc", "pathod"] +} +if platform.system() == "Windows": + BDISTS["mitmproxy"].remove("mitmproxy") + +TOOLS = [ + tool + for tools in sorted(BDISTS.values()) + for tool in tools +] def git(args: str) -> str: @@ -25,8 +81,50 @@ def git(args: str) -> str: def get_version(dev: bool = False, build: bool = False) -> str: - x = runpy.run_path(VERSION_FILE) - return x["get_version"](dev, build, True) + version = runpy.run_path(VERSION_FILE)["VERSION"] + version = re.sub(r"\.dev.+?$", "", version) # replace dev suffix if present. + + last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2) + commit = commit.lstrip("g")[:7] + tag_dist = int(tag_dist) + + if tag_dist > 0 and dev: + dev_tag = ".dev{tag_dist:04}".format(tag_dist=tag_dist) + else: + dev_tag = "" + + if tag_dist > 0 and build: + # The wheel build tag (we use the commit) must start with a digit, so we include "0x" + build_tag = "-0x{commit}".format(commit=commit) + else: + build_tag = "" + + return version + dev_tag + build_tag + + +def set_version(dev: bool) -> None: + """ + Update version information in mitmproxy's version.py to either include the dev version or not. + """ + v = get_version(dev) + with open(VERSION_FILE) as f: + content = f.read() + content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(v), content) + with open(VERSION_FILE, "w") as f: + f.write(content) + + +def archive_name(bdist: str) -> str: + if platform.system() == "Windows": + ext = "zip" + else: + ext = "tar.gz" + return "{project}-{version}-{platform}.{ext}".format( + project=bdist, + version=get_version(), + platform=PLATFORM_TAG, + ext=ext + ) def wheel_name() -> str: @@ -35,6 +133,19 @@ def wheel_name() -> str: ) +def installer_name() -> str: + ext = { + "Windows": "exe", + "Darwin": "dmg", + "Linux": "run" + }[platform.system()] + return "mitmproxy-{version}-{platform}-installer.{ext}".format( + version=get_version(), + platform=PLATFORM_TAG, + ext=ext, + ) + + @contextlib.contextmanager def chdir(path: str): old_dir = os.getcwd() @@ -51,6 +162,24 @@ def cli(): pass +@cli.command("encrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def encrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.encrypt(infile.read())) + + +@cli.command("decrypt") +@click.argument('infile', type=click.File('rb')) +@click.argument('outfile', type=click.File('wb')) +@click.argument('key', envvar='RTOOL_KEY') +def decrypt(infile, outfile, key): + f = cryptography.fernet.Fernet(key.encode()) + outfile.write(f.decrypt(infile.read())) + + @cli.command("contributors") def contributors(): """ @@ -63,31 +192,184 @@ def contributors(): f.write(contributors_data.encode()) -@cli.command("homebrew-pr") -def homebrew_pr(): +@cli.command("wheel") +def make_wheel(): """ - Create a new Homebrew PR + Build a Python wheel """ - if platform.system() != "Darwin": - print("You need to run this on macOS to create a new Homebrew PR. Sorry.") - sys.exit(1) + set_version(True) + try: + subprocess.check_call([ + "tox", "-e", "wheel", + ], env={ + **os.environ, + "VERSION": get_version(True), + }) + finally: + set_version(False) - print("Creating a new PR with Homebrew...") + +@cli.command("bdist") +def make_bdist(): + """ + Build a binary distribution + """ + if exists(PYINSTALLER_TEMP): + shutil.rmtree(PYINSTALLER_TEMP) + if exists(PYINSTALLER_DIST): + shutil.rmtree(PYINSTALLER_DIST) + + os.makedirs(DIST_DIR, exist_ok=True) + + for bdist, tools in sorted(BDISTS.items()): + with Archive(join(DIST_DIR, archive_name(bdist))) as archive: + for tool in tools: + # We can't have a folder and a file with the same name. + if tool == "mitmproxy": + tool = "mitmproxy_main" + # This is PyInstaller, so it messes up paths. + # We need to make sure that we are in the spec folder. + with chdir(PYINSTALLER_SPEC): + print("Building %s binary..." % tool) + excludes = [] + if tool != "mitmweb": + excludes.append("mitmproxy.tools.web") + if tool != "mitmproxy_main": + excludes.append("mitmproxy.tools.console") + + # Overwrite mitmproxy/version.py to include commit info + set_version(True) + try: + subprocess.check_call( + [ + "pyinstaller", + "--clean", + "--workpath", PYINSTALLER_TEMP, + "--distpath", PYINSTALLER_DIST, + "--additional-hooks-dir", PYINSTALLER_HOOKS, + "--onefile", + "--console", + "--icon", "icon.ico", + # This is PyInstaller, so setting a + # different log level obviously breaks it :-) + # "--log-level", "WARN", + ] + + [x for e in excludes for x in ["--exclude-module", e]] + + PYINSTALLER_ARGS + + [tool] + ) + finally: + set_version(False) + # Delete the spec file - we're good without. + os.remove("{}.spec".format(tool)) + + # Test if it works at all O:-) + executable = join(PYINSTALLER_DIST, tool) + if platform.system() == "Windows": + executable += ".exe" + + # Remove _main suffix from mitmproxy executable + if "_main" in executable: + shutil.move( + executable, + executable.replace("_main", "") + ) + executable = executable.replace("_main", "") + + print("> %s --version" % executable) + print(subprocess.check_output([executable, "--version"]).decode()) + + archive.add(executable, basename(executable)) + print("Packed {}.".format(archive_name(bdist))) + + +@cli.command("upload-release") +@click.option('--username', prompt=True) +@click.password_option(confirmation_prompt=False) +@click.option('--repository', default="pypi") +def upload_release(username, password, repository): + """ + Upload wheels to PyPI + """ + filename = wheel_name() + print("Uploading {} to {}...".format(filename, repository)) subprocess.check_call([ - "brew", - "bump-formula-pr", - "--url", "https://github.com/mitmproxy/mitmproxy/archive/v{}.tar.gz".format(get_version()), - "mitmproxy", + "twine", + "upload", + "-u", username, + "-p", password, + "-r", repository, + join(DIST_DIR, filename) ]) -@cli.command("encrypt") -@click.argument('infile', type=click.File('rb')) -@click.argument('outfile', type=click.File('wb')) -@click.argument('key', envvar='RTOOL_KEY') -def encrypt(infile, outfile, key): - f = cryptography.fernet.Fernet(key.encode()) - outfile.write(f.encrypt(infile.read())) +@cli.command("upload-snapshot") +@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True) +@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22) +@click.option("--user", envvar="SNAPSHOT_USER", prompt=True) +@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem")) +@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True) +@click.option("--wheel/--no-wheel", default=False) +@click.option("--bdist/--no-bdist", default=False) +@click.option("--installer/--no-installer", default=False) +def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer): + """ + Upload snapshot to snapshot server + """ + with pysftp.Connection(host=host, + port=port, + username=user, + private_key=private_key, + private_key_pass=private_key_password) as sftp: + dir_name = "snapshots/v{}".format(get_version()) + sftp.makedirs(dir_name) + with sftp.cd(dir_name): + files = [] + if wheel: + files.append(wheel_name()) + if bdist: + for bdist in sorted(BDISTS.keys()): + files.append(archive_name(bdist)) + if installer: + files.append(installer_name()) + + for f in files: + local_path = join(DIST_DIR, f) + remote_filename = re.sub( + r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()), + get_version(True, True), + f + ) + symlink_path = "../{}".format(f.replace(get_version(), "latest")) + + # Upload new version + print("Uploading {} as {}...".format(f, remote_filename)) + with click.progressbar(length=os.stat(local_path).st_size) as bar: + # We hide the file during upload + sftp.put( + local_path, + "." + remote_filename, + callback=lambda done, total: bar.update(done - bar.pos) + ) + + # Delete old versions + old_version = f.replace(get_version(), "*") + for f_old in sftp.listdir(): + if fnmatch.fnmatch(f_old, old_version): + print("Removing {}...".format(f_old)) + sftp.remove(f_old) + + # Show new version + sftp.rename("." + remote_filename, remote_filename) + + # update symlink for the latest release + if sftp.lexists(symlink_path): + print("Removing {}...".format(symlink_path)) + sftp.remove(symlink_path) + if f != wheel_name(): + # "latest" isn't a proper wheel version, so this could not be installed. + # https://github.com/mitmproxy/mitmproxy/issues/1065 + sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path) if __name__ == "__main__": diff --git a/tox.ini b/tox.ini index a9054e5b2..3ec5a3874 100644 --- a/tox.ini +++ b/tox.ini @@ -33,15 +33,27 @@ commands = python ./test/individual_coverage.py [testenv:cibuild] -passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL +passenv = TRAVIS_* AWS_* APPVEYOR_* TWINE_* RTOOL_KEY WHEEL deps = -rrequirements.txt pyinstaller==3.3.1 + twine==1.11.0 awscli commands = mitmdump --version python ./release/ci.py {posargs} +[testenv:wheeltest] +recreate = True +deps = +commands = + pip install {posargs} + mitmproxy --version + mitmdump --version + mitmweb --version + pathod --version + pathoc --version + [testenv:docs] passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL deps =