From 20d89cd34fa89ceaceab962bc15dac2c0c59d702 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Aug 2015 12:39:06 +1200 Subject: [PATCH 01/30] Initial checkin --- .env | 5 ++++ .gitignore | 15 ++++++++++ README | 10 +++++++ osx-binaries | 66 ++++++++++++++++++++++++++++++++++++++++++++ release-checklist.md | 55 ++++++++++++++++++++++++++++++++++++ requirements.txt | 2 ++ test-release | 38 +++++++++++++++++++++++++ 7 files changed, 191 insertions(+) create mode 100644 .env create mode 100644 .gitignore create mode 100644 README create mode 100755 osx-binaries create mode 100644 release-checklist.md create mode 100644 requirements.txt create mode 100755 test-release diff --git a/.env b/.env new file mode 100644 index 000000000..97f38452b --- /dev/null +++ b/.env @@ -0,0 +1,5 @@ +DIR="${0%/*}" +if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then + echo "Activating mitmproxy virtualenv..." + source "$DIR/../venv.mitmproxy/bin/activate" +fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..5bb3661e1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +.DS_Store +MANIFEST +/build +/dist +/tmp +/doc +/venv +/libmproxy/gui +/release/build +*.py[cdo] +*.swp +*.swo + +/venv +/release diff --git a/README b/README new file mode 100644 index 000000000..58e917f0f --- /dev/null +++ b/README @@ -0,0 +1,10 @@ + +General build and release utilities for the mitmproxy, netlib and pathod +projects. These tools assume a directory structure with all repositories at the +same level, for example: + + /src/ + ./mitmproxy + ./netlib + ./pathod + ./release diff --git a/osx-binaries b/osx-binaries new file mode 100755 index 000000000..3cfe0913b --- /dev/null +++ b/osx-binaries @@ -0,0 +1,66 @@ +#!/bin/sh + +# Quick and dangerous script for building OSX binaries. + +# At the moment, pyinstaller has no support for entry points, except for this +# hideous hack on the wiki: +# https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Setuptools-Entry-Point +# Once this is fixed, we can ditch the redundant command scripts. + +VENV=../venv.mitmproxy +PYINST_CMD="$VENV/bin/pyinstaller -F --clean" +TMPDIR=./tmp +CACHE="~/Library/Application Support/pyinstaller" + + +if [ ! -d $VENV ] +then + echo "Failed: set up a dev environment as described in the README" + echo "and run from the top-level mitmproxy directory." + exit +fi + +source $VENV/bin/activate + +if [ ! -f $VENV/bin/pyinstaller ] + then + echo "Installing pyinstaller..." + $VENV/bin/pip install \ + --force-reinstall \ + --upgrade \ + https://github.com/pyinstaller/pyinstaller/archive/develop.zip + $VENV/bin/pip install --upgrade macholib +fi + +# readline.so is actually a symlink to a Python file, which breaks PyInstaller +# (and readline itself). Why? Who knows. Re-address this when this stupidity +# ceases to be. +echo "Removing broken readline..." +rm -f $VENV/lib/python2.7/readline.so + + +echo "Clearing caches..." +rm -f dist/* +rm -rf $TMPDIR +rm -rf $CACHE + +$PYINST_CMD ./release/mitmdump.spec +echo "Running mitmdump..." +./dist/mitmdump --version || exit 1 + +$PYINST_CMD ./release/mitmproxy.spec +echo "Running mitmproxy..." +./dist/mitmproxy --version || exit 1 + +$PYINST_CMD ./release/mitmweb.spec +echo "Running mitmweb..." +./dist/mitmweb --version || exit 1 + +DST=osx-mitmproxy-`./dist/mitmdump --shortversion 2>&1` +mkdir -p $TMPDIR/$DST +cp ./dist/mitmproxy $TMPDIR/$DST +cp ./dist/mitmdump $TMPDIR/$DST +cshape ./doc-src $TMPDIR/$DST/doc + +cd $TMPDIR +tar -czvf $DST.tar.gz $DST diff --git a/release-checklist.md b/release-checklist.md new file mode 100644 index 000000000..e6d9ae1f4 --- /dev/null +++ b/release-checklist.md @@ -0,0 +1,55 @@ +# Release Checklist + +## Test + + - Create the source distributions, make sure the output is sensible: + `./release/build.py release` + All source distributions can be found in `./dist`. + + - Test the source distributions: + `./release/build.py test` + This creates a new virtualenv in `../venv.mitmproxy-release` and installs the distributions from `./dist` into it. + +## Release + + - Verify that repositories are in a clean state: + `./release/build.py git status` + + - Update the version number in `version.py` for all projects: + `./release/build.py set-version 0.13` + + - Ensure that the website style assets have been compiled for production, and synced to the docs. + + - Render the docs, update CONTRIBUTORS file: + `./release/build.py docs contributors` + + - Make version bump commit for all projects, tag and push it: + `./release/build.py git commit -am "bump version"` + `./release/build.py git tag v0.13` + `./release/build.py git push --tags` + + - Recreate the source distributions with updated version information: + `./release/build.py sdist` + + - Build the OSX binaries + - Follow instructions in osx-binaries + - Move to download dir: + `mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download` + + - Move all source distributions from `./dist` to the server: + `mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download` + + - Upload distributions in `./dist` to PyPI: + `./release/build.py upload` + You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`. + ([more info](https://tom-christie.github.io/articles/pypi/)) + + - Now bump the version number to be ready for the next cycle: + + **TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? + We should probably just leave it as-is and only bump once we actually do the next release. + + Also, we need a release policy. I propose the following: + - By default, every release is a new minor (`0.x`) release and it will be pushed for all three projects. + - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release for a single project. + This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)` \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..de1b90b2a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +click>=4.1 +twine>=1.5.0 diff --git a/test-release b/test-release new file mode 100755 index 000000000..8cbcea8c5 --- /dev/null +++ b/test-release @@ -0,0 +1,38 @@ +#!/bin/bash + +MITMPROXY_DIR=~/mitmproxy/mitmproxy +NETLIB_DIR=~/mitmproxy/netlib +PATHOD_DIR=~/mitmproxy/pathod +DST=/tmp/mitmproxy_release + +rm -rf $DST +mkdir -p $DST + +cd $NETLIB_DIR +echo "Creating netlib source distribution..." +python ./setup.py -q sdist --dist-dir $DST + +echo "Creating mitmproxy source distribution..." +cd $MITMPROXY_DIR +python ./setup.py -q sdist --dist-dir $DST + +echo "Creating pathod source distribution..." +cd $PATHOD_DIR +python ./setup.py -q sdist --dist-dir $DST + +echo "Creating virtualenv for test install..." +virtualenv -q $DST/venv + +cd $DST +echo "Installing netlib..." +./venv/bin/pip -q install --download-cache ~/.pipcache ./netlib* +echo "Installing pathod..." +./venv/bin/pip -q install --download-cache ~/.pipcache ./pathod* +echo "Installing mitmproxy..." +./venv/bin/pip -q install --download-cache ~/.pipcache ./mitmproxy* + +echo "Running binaries..." +./venv/bin/mitmproxy --version +./venv/bin/mitmdump --version +./venv/bin/pathod --version +./venv/bin/pathoc --version From c3fffe152ec2b09e3cd079ff59dc1602047fd11b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Aug 2015 12:51:52 +1200 Subject: [PATCH 02/30] Build script, extension of contributors command, .gitignore --- .gitignore | 4 - build | 285 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 285 insertions(+), 4 deletions(-) create mode 100755 build diff --git a/.gitignore b/.gitignore index 5bb3661e1..356e5994d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,8 @@ .DS_Store MANIFEST -/build /dist /tmp -/doc /venv -/libmproxy/gui -/release/build *.py[cdo] *.swp *.swo diff --git a/build b/build new file mode 100755 index 000000000..da88d9818 --- /dev/null +++ b/build @@ -0,0 +1,285 @@ +#!/usr/bin/env python +from __future__ import ( + absolute_import, print_function, division, unicode_literals +) +from contextlib import contextmanager +from os.path import dirname, realpath, join, exists, normpath +import os +import shutil +import subprocess +import glob +import re +import shlex +import click + +# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes +# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ +if os.name == "nt": + venv_bin = "Scripts" +else: + venv_bin = "bin" + +release_dir = join(dirname(realpath(__file__))) +root_dir = join(release_dir, "..") +mitmproxy_dir = join(root_dir, "mitmproxy") +dist_dir = join(mitmproxy_dir, "dist") +test_venv_dir = join(release_dir, "venv") + +all_projects = ("netlib", "pathod", "mitmproxy") +tools = { + "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], + "pathod": ["pathod", "pathoc"], + "netlib": [] +} +if os.name == "nt": + tools["mitmproxy"].remove("mitmproxy") +version_files = { + "mitmproxy": normpath(join(root_dir, "mitmproxy/libmproxy/version.py")), + "pathod": normpath(join(root_dir, "pathod/libpathod/version.py")), + "netlib": normpath(join(root_dir, "netlib/netlib/version.py")), +} + + +@contextmanager +def empty_pythonpath(): + """ + Make sure that the regular python installation is not on the python path, + which would give us access to modules installed outside of our virtualenv. + """ + pythonpath = os.environ["PYTHONPATH"] + os.environ["PYTHONPATH"] = "" + yield + os.environ["PYTHONPATH"] = pythonpath + + +@contextmanager +def chdir(path): + old_dir = os.getcwd() + os.chdir(path) + yield + os.chdir(old_dir) + + +@click.group(chain=True) +def cli(): + """ + mitmproxy build tool + """ + pass + + +@cli.command("contributors") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(all_projects), default=all_projects +) +def contributors(projects): + """ + Update CONTRIBUTORS.md + """ + for project in all_projects: + if project not in projects: + continue + with chdir(os.path.join(root_dir, project)): + print("Updating %s/CONTRIBUTORS..."%project) + contributors_data = subprocess.check_output( + shlex.split("git shortlog -n -s") + ) + with open("CONTRIBUTORS", "w+") as f: + f.write(contributors_data) + + +@cli.command("docs") +def docs(): + """ + Render the docs + """ + print("Rendering the docs...") + subprocess.check_call([ + "cshape", + join(mitmproxy_dir, "doc-src"), + join(mitmproxy_dir, "doc") + ]) + + +@cli.command("set-version") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(all_projects), default=all_projects +) +@click.argument('version') +def set_version(projects, version): + """ + Update version information + """ + print("Update versions...") + version = ", ".join(version.split(".")) + for project, version_file in version_files.items(): + if project not in projects: + continue + print("Update %s..." % version_file) + with open(version_file, "rb") as f: + content = f.read() + new_content = re.sub( + r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, + content + ) + with open(version_file, "wb") as f: + f.write(new_content) + + +@cli.command("git") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(all_projects), default=all_projects +) +@click.argument('args', nargs=-1, required=True) +def git(projects, args): + """ + Run a git command on every project + """ + args = ["git"] + list(args) + for project in projects: + print("%s> %s..." % (project, " ".join(args))) + subprocess.check_call( + args, + cwd=join(root_dir, project) + ) + + +@cli.command("sdist") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(all_projects), default=all_projects +) +def sdist(projects): + """ + Build a source distribution + """ + with empty_pythonpath(): + print("Building release...") + if exists(dist_dir): + shutil.rmtree(dist_dir) + for project in projects: + print("Creating %s source distribution..." % project) + subprocess.check_call( + [ + "python", "./setup.py", + "-q", "sdist", "--dist-dir", dist_dir, "--formats=gztar" + ], + cwd=join(root_dir, project) + ) + + +@cli.command("test") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(all_projects), default=all_projects +) +@click.pass_context +def test(ctx, projects): + """ + Test the source distribution + """ + if not exists(dist_dir): + ctx.invoke(sdist) + + with empty_pythonpath(): + print("Creating virtualenv for test install...") + if exists(test_venv_dir): + shutil.rmtree(test_venv_dir) + subprocess.check_call(["virtualenv", "-q", test_venv_dir]) + + pip = join(test_venv_dir, venv_bin, "pip") + with chdir(dist_dir): + for project in projects: + print("Installing %s..." % project) + dist = join(root_dir, project) + subprocess.check_call([pip, "install", "-q", dist]) + + print("Running binaries...") + for project in projects: + for tool in tools[project]: + tool = join(test_venv_dir, venv_bin, tool) + print(tool) + print(subprocess.check_output([tool, "--version"])) + + print("Virtualenv available for further testing:") + print( + "source %s" % normpath( + join(test_venv_dir, venv_bin, "activate") + ) + ) + + +@cli.command("upload") +@click.option('--username', prompt=True) +@click.password_option(confirmation_prompt=False) +@click.option('--repository', default="pypi") +def upload_release(username, password, repository): + """ + Upload source distributions to PyPI + """ + print("Uploading distributions...") + subprocess.check_call([ + "twine", + "upload", + "-u", username, + "-p", password, + "-r", repository, + "%s/*" % dist_dir + ]) + + +# TODO: Fully automate build process. +# This wizard is missing OSX builds and updating mitmproxy.org. +@cli.command("wizard") +@click.option('--version', prompt=True) +@click.option('--username', prompt="PyPI Username") +@click.password_option(confirmation_prompt=False, prompt="PyPI Password") +@click.option('--repository', default="pypi") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(all_projects), default=all_projects +) +@click.pass_context +def wizard(ctx, version, username, password, repository, projects): + """ + Interactive Release Wizard + """ + for project in projects: + if subprocess.check_output( + ["git", "status", "--porcelain"], cwd=join(root_dir, project) + ): + raise RuntimeError("%s repository is not clean." % project) + + # Build test release + ctx.invoke(sdist, projects=projects) + ctx.invoke(test, projects=projects) + click.confirm("Please test the release now. Is it ok?", abort=True) + + # bump version, update docs and contributors + ctx.invoke(set_version, version=version, projects=projects) + ctx.invoke(docs) + ctx.invoke(contributors) + + # version bump commit + tag + ctx.invoke( + git, args=["commit", "-a", "-m", "bump version"], projects=projects + ) + ctx.invoke(git, args=["tag", "v" + version], projects=projects) + ctx.invoke(git, args=["push"], projects=projects) + ctx.invoke(git, args=["push", "--tags"], projects=projects) + + # Re-invoke sdist with bumped version + ctx.invoke(sdist, projects=projects) + click.confirm("All good, can upload to PyPI?", abort=True) + ctx.invoke( + upload_release, + username=username, password=password, repository=repository + ) + click.echo("All done!") + + +if __name__ == "__main__": + cli() From 5b957ac6583e60e2c0ecef0a44f60ecf44c17fad Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Aug 2015 12:57:24 +1200 Subject: [PATCH 03/30] constants -> CONSTANTS --- build | 84 +++++++++++++++++++++++++++++------------------------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/build b/build index da88d9818..49e3213f1 100755 --- a/build +++ b/build @@ -15,28 +15,28 @@ import click # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ if os.name == "nt": - venv_bin = "Scripts" + VENV_BIN = "Scripts" else: - venv_bin = "bin" + VENV_BIN = "bin" -release_dir = join(dirname(realpath(__file__))) -root_dir = join(release_dir, "..") -mitmproxy_dir = join(root_dir, "mitmproxy") -dist_dir = join(mitmproxy_dir, "dist") -test_venv_dir = join(release_dir, "venv") +RELEASE_DIR = join(dirname(realpath(__file__))) +ROOT_DIR = join(RELEASE_DIR, "..") +MITMPROXY_DIR = join(ROOT_DIR, "mitmproxy") +DIST_DIR = join(MITMPROXY_DIR, "dist") +TEST_VENV_DIR = join(RELEASE_DIR, "venv") -all_projects = ("netlib", "pathod", "mitmproxy") -tools = { +PROJECTS = ("netlib", "pathod", "mitmproxy") +TOOLS = { "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], "pathod": ["pathod", "pathoc"], "netlib": [] } if os.name == "nt": - tools["mitmproxy"].remove("mitmproxy") -version_files = { - "mitmproxy": normpath(join(root_dir, "mitmproxy/libmproxy/version.py")), - "pathod": normpath(join(root_dir, "pathod/libpathod/version.py")), - "netlib": normpath(join(root_dir, "netlib/netlib/version.py")), + TOOLS["mitmproxy"].remove("mitmproxy") +VERSION_FILES = { + "mitmproxy": normpath(join(ROOT_DIR, "mitmproxy/libmproxy/version.py")), + "pathod": normpath(join(ROOT_DIR, "pathod/libpathod/version.py")), + "netlib": normpath(join(ROOT_DIR, "netlib/netlib/version.py")), } @@ -71,16 +71,16 @@ def cli(): @cli.command("contributors") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(all_projects), default=all_projects + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) def contributors(projects): """ Update CONTRIBUTORS.md """ - for project in all_projects: + for project in PROJECTS: if project not in projects: continue - with chdir(os.path.join(root_dir, project)): + with chdir(os.path.join(ROOT_DIR, project)): print("Updating %s/CONTRIBUTORS..."%project) contributors_data = subprocess.check_output( shlex.split("git shortlog -n -s") @@ -97,15 +97,15 @@ def docs(): print("Rendering the docs...") subprocess.check_call([ "cshape", - join(mitmproxy_dir, "doc-src"), - join(mitmproxy_dir, "doc") + join(MITMPROXY_DIR, "doc-src"), + join(MITMPROXY_DIR, "doc") ]) @cli.command("set-version") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(all_projects), default=all_projects + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) @click.argument('version') def set_version(projects, version): @@ -114,7 +114,7 @@ def set_version(projects, version): """ print("Update versions...") version = ", ".join(version.split(".")) - for project, version_file in version_files.items(): + for project, version_file in VERSION_FILES.items(): if project not in projects: continue print("Update %s..." % version_file) @@ -131,7 +131,7 @@ def set_version(projects, version): @cli.command("git") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(all_projects), default=all_projects + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) @click.argument('args', nargs=-1, required=True) def git(projects, args): @@ -143,14 +143,14 @@ def git(projects, args): print("%s> %s..." % (project, " ".join(args))) subprocess.check_call( args, - cwd=join(root_dir, project) + cwd=join(ROOT_DIR, project) ) @cli.command("sdist") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(all_projects), default=all_projects + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) def sdist(projects): """ @@ -158,56 +158,56 @@ def sdist(projects): """ with empty_pythonpath(): print("Building release...") - if exists(dist_dir): - shutil.rmtree(dist_dir) + if exists(DIST_DIR): + shutil.rmtree(DIST_DIR) for project in projects: print("Creating %s source distribution..." % project) subprocess.check_call( [ "python", "./setup.py", - "-q", "sdist", "--dist-dir", dist_dir, "--formats=gztar" + "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar" ], - cwd=join(root_dir, project) + cwd=join(ROOT_DIR, project) ) @cli.command("test") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(all_projects), default=all_projects + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) @click.pass_context def test(ctx, projects): """ Test the source distribution """ - if not exists(dist_dir): + if not exists(DIST_DIR): ctx.invoke(sdist) with empty_pythonpath(): print("Creating virtualenv for test install...") - if exists(test_venv_dir): - shutil.rmtree(test_venv_dir) - subprocess.check_call(["virtualenv", "-q", test_venv_dir]) + if exists(TEST_VENV_DIR): + shutil.rmtree(TEST_VENV_DIR) + subprocess.check_call(["virtualenv", "-q", TEST_VENV_DIR]) - pip = join(test_venv_dir, venv_bin, "pip") - with chdir(dist_dir): + pip = join(TEST_VENV_DIR, VENV_BIN, "pip") + with chdir(DIST_DIR): for project in projects: print("Installing %s..." % project) - dist = join(root_dir, project) + dist = join(ROOT_DIR, project) subprocess.check_call([pip, "install", "-q", dist]) print("Running binaries...") for project in projects: - for tool in tools[project]: - tool = join(test_venv_dir, venv_bin, tool) + for tool in TOOLS[project]: + tool = join(TEST_VENV_DIR, VENV_BIN, tool) print(tool) print(subprocess.check_output([tool, "--version"])) print("Virtualenv available for further testing:") print( "source %s" % normpath( - join(test_venv_dir, venv_bin, "activate") + join(TEST_VENV_DIR, VENV_BIN, "activate") ) ) @@ -227,7 +227,7 @@ def upload_release(username, password, repository): "-u", username, "-p", password, "-r", repository, - "%s/*" % dist_dir + "%s/*" % DIST_DIR ]) @@ -240,7 +240,7 @@ def upload_release(username, password, repository): @click.option('--repository', default="pypi") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(all_projects), default=all_projects + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) @click.pass_context def wizard(ctx, version, username, password, repository, projects): @@ -249,7 +249,7 @@ def wizard(ctx, version, username, password, repository, projects): """ for project in projects: if subprocess.check_output( - ["git", "status", "--porcelain"], cwd=join(root_dir, project) + ["git", "status", "--porcelain"], cwd=join(ROOT_DIR, project) ): raise RuntimeError("%s repository is not clean." % project) From b1908e6639c9ced1ef8ce2e6d8d51a073e1ee89c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Aug 2015 17:25:38 +1200 Subject: [PATCH 04/30] Updates build -> rtool to avoid clashing with standard pyinstaller dirs checklist updates script adaptations --- .gitignore | 1 + osx-binaries | 33 ++++++++++++------------- release-checklist.md | 57 ++++++++++++++++++++++++-------------------- requirements.txt | 1 + build => rtool | 35 +++++++++++++-------------- 5 files changed, 67 insertions(+), 60 deletions(-) rename build => rtool (91%) diff --git a/.gitignore b/.gitignore index 356e5994d..dfd657451 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ MANIFEST /venv /release +/build diff --git a/osx-binaries b/osx-binaries index 3cfe0913b..657dbae93 100755 --- a/osx-binaries +++ b/osx-binaries @@ -7,10 +7,11 @@ # https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Setuptools-Entry-Point # Once this is fixed, we can ditch the redundant command scripts. -VENV=../venv.mitmproxy +VENV=./venv PYINST_CMD="$VENV/bin/pyinstaller -F --clean" TMPDIR=./tmp CACHE="~/Library/Application Support/pyinstaller" +MITMPROXY="../mitmproxy" if [ ! -d $VENV ] @@ -44,23 +45,23 @@ rm -f dist/* rm -rf $TMPDIR rm -rf $CACHE -$PYINST_CMD ./release/mitmdump.spec -echo "Running mitmdump..." -./dist/mitmdump --version || exit 1 +# $PYINST_CMD $MITMPROXY/release/mitmdump.spec +# echo "Running mitmdump..." +# ./dist/mitmdump --version || exit 1 -$PYINST_CMD ./release/mitmproxy.spec +$PYINST_CMD $MITMPROXY/release/mitmproxy.spec echo "Running mitmproxy..." ./dist/mitmproxy --version || exit 1 -$PYINST_CMD ./release/mitmweb.spec -echo "Running mitmweb..." -./dist/mitmweb --version || exit 1 +# $PYINST_CMD $MITMPROXY/release/mitmweb.spec +# echo "Running mitmweb..." +# ./dist/mitmweb --version || exit 1 -DST=osx-mitmproxy-`./dist/mitmdump --shortversion 2>&1` -mkdir -p $TMPDIR/$DST -cp ./dist/mitmproxy $TMPDIR/$DST -cp ./dist/mitmdump $TMPDIR/$DST -cshape ./doc-src $TMPDIR/$DST/doc - -cd $TMPDIR -tar -czvf $DST.tar.gz $DST +# DST=osx-mitmproxy-`./dist/mitmdump --shortversion 2>&1` +# mkdir -p $TMPDIR/$DST +# cp ./dist/mitmproxy $TMPDIR/$DST +# cp ./dist/mitmdump $TMPDIR/$DST +# cshape ./doc-src $TMPDIR/$DST/doc +# +# cd $TMPDIR +# tar -czvf $DST.tar.gz $DST diff --git a/release-checklist.md b/release-checklist.md index e6d9ae1f4..59fa7623d 100644 --- a/release-checklist.md +++ b/release-checklist.md @@ -1,35 +1,37 @@ # Release Checklist -## Test +## Check out release versions - - Create the source distributions, make sure the output is sensible: - `./release/build.py release` - All source distributions can be found in `./dist`. - - - Test the source distributions: - `./release/build.py test` - This creates a new virtualenv in `../venv.mitmproxy-release` and installs the distributions from `./dist` into it. - -## Release + - Check out the versions of pathod, netlib and mitmproxy due to be released - Verify that repositories are in a clean state: - `./release/build.py git status` - - - Update the version number in `version.py` for all projects: - `./release/build.py set-version 0.13` + `./build git status` - Ensure that the website style assets have been compiled for production, and synced to the docs. - Render the docs, update CONTRIBUTORS file: - `./release/build.py docs contributors` - - - Make version bump commit for all projects, tag and push it: - `./release/build.py git commit -am "bump version"` - `./release/build.py git tag v0.13` - `./release/build.py git push --tags` + `./build docs contributors` - - Recreate the source distributions with updated version information: - `./release/build.py sdist` + +## Test + + - Test the source distributions: + + `./build test` + + This does the following: + - creates a venv in release/venv + - creates source distributions in release/release + - installs the source distributions in the venv + - and runs all installed tools + + +## Release + + - Make a release commit for all projects, tag and push it: + `./build git commit -am "Release v0.13"` + `./build git tag v0.13` + `./build git push --tags` - Build the OSX binaries - Follow instructions in osx-binaries @@ -40,16 +42,19 @@ `mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download` - Upload distributions in `./dist` to PyPI: - `./release/build.py upload` + `./build upload` You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`. ([more info](https://tom-christie.github.io/articles/pypi/)) - Now bump the version number to be ready for the next cycle: - **TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? + **TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? We should probably just leave it as-is and only bump once we actually do the next release. - + + - Bump the version number in `version.py` for all projects: + `./build set-version 0.13` + Also, we need a release policy. I propose the following: - By default, every release is a new minor (`0.x`) release and it will be pushed for all three projects. - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release for a single project. - This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)` \ No newline at end of file + This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)` diff --git a/requirements.txt b/requirements.txt index de1b90b2a..0bb2e3785 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ click>=4.1 twine>=1.5.0 +pyinstaller diff --git a/build b/rtool similarity index 91% rename from build rename to rtool index 49e3213f1..356ea963b 100755 --- a/build +++ b/rtool @@ -2,8 +2,9 @@ from __future__ import ( absolute_import, print_function, division, unicode_literals ) -from contextlib import contextmanager -from os.path import dirname, realpath, join, exists, normpath +from os.path import join +import contextlib +import os.path import os import shutil import subprocess @@ -19,10 +20,10 @@ if os.name == "nt": else: VENV_BIN = "bin" -RELEASE_DIR = join(dirname(realpath(__file__))) +RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) +DIST_DIR = join(RELEASE_DIR, "release") ROOT_DIR = join(RELEASE_DIR, "..") MITMPROXY_DIR = join(ROOT_DIR, "mitmproxy") -DIST_DIR = join(MITMPROXY_DIR, "dist") TEST_VENV_DIR = join(RELEASE_DIR, "venv") PROJECTS = ("netlib", "pathod", "mitmproxy") @@ -34,13 +35,13 @@ TOOLS = { if os.name == "nt": TOOLS["mitmproxy"].remove("mitmproxy") VERSION_FILES = { - "mitmproxy": normpath(join(ROOT_DIR, "mitmproxy/libmproxy/version.py")), - "pathod": normpath(join(ROOT_DIR, "pathod/libpathod/version.py")), - "netlib": normpath(join(ROOT_DIR, "netlib/netlib/version.py")), + "mitmproxy": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"), + "pathod": join(ROOT_DIR, "pathod/libpathod/version.py"), + "netlib": join(ROOT_DIR, "netlib/netlib/version.py"), } -@contextmanager +@contextlib.contextmanager def empty_pythonpath(): """ Make sure that the regular python installation is not on the python path, @@ -52,7 +53,7 @@ def empty_pythonpath(): os.environ["PYTHONPATH"] = pythonpath -@contextmanager +@contextlib.contextmanager def chdir(path): old_dir = os.getcwd() os.chdir(path) @@ -158,7 +159,7 @@ def sdist(projects): """ with empty_pythonpath(): print("Building release...") - if exists(DIST_DIR): + if os.path.exists(DIST_DIR): shutil.rmtree(DIST_DIR) for project in projects: print("Creating %s source distribution..." % project) @@ -171,22 +172,20 @@ def sdist(projects): ) -@cli.command("test") +@cli.command("mkvenv") @click.option( '--project', '-p', 'projects', multiple=True, type=click.Choice(PROJECTS), default=PROJECTS ) @click.pass_context -def test(ctx, projects): +def mkvenv(ctx, projects): """ - Test the source distribution + make a venv and test the source distribution """ - if not exists(DIST_DIR): - ctx.invoke(sdist) - + ctx.invoke(sdist) with empty_pythonpath(): print("Creating virtualenv for test install...") - if exists(TEST_VENV_DIR): + if os.path.exists(TEST_VENV_DIR): shutil.rmtree(TEST_VENV_DIR) subprocess.check_call(["virtualenv", "-q", TEST_VENV_DIR]) @@ -206,7 +205,7 @@ def test(ctx, projects): print("Virtualenv available for further testing:") print( - "source %s" % normpath( + "source %s" % os.path.normpath( join(TEST_VENV_DIR, VENV_BIN, "activate") ) ) From 11c626cf8418dbac1c6aea701a298e3c166b4721 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Aug 2015 20:09:33 +1200 Subject: [PATCH 05/30] OSX binary builds in Python script --- .gitignore | 1 + README | 83 +++++++++++++++++++++++++++++++++++++++++--- osx-binaries | 7 ---- release-checklist.md | 60 -------------------------------- requirements.txt | 1 - rtool | 70 ++++++++++++++++++++++++++++++++----- 6 files changed, 141 insertions(+), 81 deletions(-) delete mode 100644 release-checklist.md diff --git a/.gitignore b/.gitignore index dfd657451..b3f4fc3e0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ MANIFEST /venv /release /build +/pyinstallerdist diff --git a/README b/README index 58e917f0f..7754125d0 100644 --- a/README +++ b/README @@ -3,8 +3,81 @@ General build and release utilities for the mitmproxy, netlib and pathod projects. These tools assume a directory structure with all repositories at the same level, for example: - /src/ - ./mitmproxy - ./netlib - ./pathod - ./release + /src + /mitmproxy + /netlib + /pathod + /release + + +# Release policies + + - By default, every release is a new minor (`0.x`) release and it will be + pushed for all three projects. + + - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release + for a single project. This matches with what we do in `setup.py`: + + "netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION) + + + +# Release Checklist + +## Check out release versions + + - Check out the versions of pathod, netlib and mitmproxy due to be released + + - Verify that repositories are in a clean state: + + `./build git status` + + - Ensure that the website style assets have been compiled for production, and + synced to the docs. + + - Render the docs, update CONTRIBUTORS file: + + ./build docs contributors + + +## Test + + - Test the source distributions: + + ./build test + + This does the following: + - creates a venv in release/venv + - creates source distributions in release/release + - installs the source distributions in the venv + - and runs all installed tools + + +## Release + + - Make a release commit for all projects, tag and push it: + + ./build git commit -am "Release v0.13" + ./build git tag v0.13 + ./build git push --tags + + - Build the OSX binaries + - Follow instructions in osx-binaries + - Move to download dir: + + mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download + + - Move all source distributions from `./dist` to the server: + + mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download + + - Upload distributions in `./dist` to PyPI: + + ./build upload + + You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`. + ([more info](https://tom-christie.github.io/articles/pypi/)) + + - Now bump the version number to be ready for the next cycle + + `./build set-version 0.13` diff --git a/osx-binaries b/osx-binaries index 657dbae93..2d1acccc7 100755 --- a/osx-binaries +++ b/osx-binaries @@ -33,13 +33,6 @@ if [ ! -f $VENV/bin/pyinstaller ] $VENV/bin/pip install --upgrade macholib fi -# readline.so is actually a symlink to a Python file, which breaks PyInstaller -# (and readline itself). Why? Who knows. Re-address this when this stupidity -# ceases to be. -echo "Removing broken readline..." -rm -f $VENV/lib/python2.7/readline.so - - echo "Clearing caches..." rm -f dist/* rm -rf $TMPDIR diff --git a/release-checklist.md b/release-checklist.md deleted file mode 100644 index 59fa7623d..000000000 --- a/release-checklist.md +++ /dev/null @@ -1,60 +0,0 @@ -# Release Checklist - -## Check out release versions - - - Check out the versions of pathod, netlib and mitmproxy due to be released - - - Verify that repositories are in a clean state: - `./build git status` - - - Ensure that the website style assets have been compiled for production, and synced to the docs. - - - Render the docs, update CONTRIBUTORS file: - `./build docs contributors` - - -## Test - - - Test the source distributions: - - `./build test` - - This does the following: - - creates a venv in release/venv - - creates source distributions in release/release - - installs the source distributions in the venv - - and runs all installed tools - - -## Release - - - Make a release commit for all projects, tag and push it: - `./build git commit -am "Release v0.13"` - `./build git tag v0.13` - `./build git push --tags` - - - Build the OSX binaries - - Follow instructions in osx-binaries - - Move to download dir: - `mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download` - - - Move all source distributions from `./dist` to the server: - `mv ./dist/* ~/mitmproxy/www.mitmproxy.org/src/download` - - - Upload distributions in `./dist` to PyPI: - `./build upload` - You can test with [testpypi.python.org](https://testpypi.python.org/pypi) by passing `--repository test`. - ([more info](https://tom-christie.github.io/articles/pypi/)) - - - Now bump the version number to be ready for the next cycle: - - **TODO**: We just shipped 0.12 - do we bump to 0.12.1 or 0.13 now? - We should probably just leave it as-is and only bump once we actually do the next release. - - - Bump the version number in `version.py` for all projects: - `./build set-version 0.13` - - Also, we need a release policy. I propose the following: - - By default, every release is a new minor (`0.x`) release and it will be pushed for all three projects. - - Only if an emergency bugfix is needed, we push a new `0.x.y` bugfix release for a single project. - This matches with what we do in `setup.py`: `"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION)` diff --git a/requirements.txt b/requirements.txt index 0bb2e3785..de1b90b2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ click>=4.1 twine>=1.5.0 -pyinstaller diff --git a/rtool b/rtool index 356ea963b..bb33604b0 100755 --- a/rtool +++ b/rtool @@ -20,11 +20,23 @@ if os.name == "nt": else: VENV_BIN = "bin" + + RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) DIST_DIR = join(RELEASE_DIR, "release") ROOT_DIR = join(RELEASE_DIR, "..") MITMPROXY_DIR = join(ROOT_DIR, "mitmproxy") -TEST_VENV_DIR = join(RELEASE_DIR, "venv") + +PYINSTALLER_URL =\ + "https://github.com/pyinstaller/pyinstaller/archive/develop.zip" +PYINSTALLER_CACHE = os.path.expanduser( + "~/Library/Application Support/pyinstaller" +) +PYINSTALLER_DIST = join(RELEASE_DIR, "pyinstallerdist") + +VENV_DIR = join(RELEASE_DIR, "venv") +VENV_PIP = join(VENV_DIR, VENV_BIN, "pip") +VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") PROJECTS = ("netlib", "pathod", "mitmproxy") TOOLS = { @@ -172,6 +184,49 @@ def sdist(projects): ) +@cli.command("osxbin") +@click.option( + '--project', '-p', 'projects', + multiple=True, type=click.Choice(PROJECTS), default=PROJECTS +) +@click.pass_context +def osxbin(ctx, projects): + if not os.path.exists(VENV_PYINSTALLER): + subprocess.check_call( + [ + VENV_PIP, + "install", + PYINSTALLER_URL + ] + ) + + shutil.rmtree(PYINSTALLER_CACHE, ignore_errors=True) + shutil.rmtree("./build", ignore_errors=True) + shutil.rmtree(PYINSTALLER_DIST, ignore_errors=True) + for p in projects: + specs = glob.glob(os.path.join(ROOT_DIR, p, "release/*.spec")) + for spec in specs: + subprocess.check_call( + [ + VENV_PYINSTALLER, + "--distpath", PYINSTALLER_DIST, + spec + ] + ) + if specs and os.path.exists(PYINSTALLER_DIST): + bins = os.listdir(PYINSTALLER_DIST) + for bin in bins: + subprocess.check_call( + [ + os.path.join(PYINSTALLER_DIST, bin), + "--version" + ] + ) + + + + + @cli.command("mkvenv") @click.option( '--project', '-p', 'projects', @@ -185,28 +240,27 @@ def mkvenv(ctx, projects): ctx.invoke(sdist) with empty_pythonpath(): print("Creating virtualenv for test install...") - if os.path.exists(TEST_VENV_DIR): - shutil.rmtree(TEST_VENV_DIR) - subprocess.check_call(["virtualenv", "-q", TEST_VENV_DIR]) + if os.path.exists(VENV_DIR): + shutil.rmtree(VENV_DIR) + subprocess.check_call(["virtualenv", "-q", VENV_DIR]) - pip = join(TEST_VENV_DIR, VENV_BIN, "pip") with chdir(DIST_DIR): for project in projects: print("Installing %s..." % project) dist = join(ROOT_DIR, project) - subprocess.check_call([pip, "install", "-q", dist]) + subprocess.check_call([VENV_PIP, "install", "-q", dist]) print("Running binaries...") for project in projects: for tool in TOOLS[project]: - tool = join(TEST_VENV_DIR, VENV_BIN, tool) + tool = join(VENV_DIR, VENV_BIN, tool) print(tool) print(subprocess.check_output([tool, "--version"])) print("Virtualenv available for further testing:") print( "source %s" % os.path.normpath( - join(TEST_VENV_DIR, VENV_BIN, "activate") + join(VENV_DIR, VENV_BIN, "activate") ) ) From 10b377bb10158f47643978d932a86e7b4c974452 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Aug 2015 20:11:17 +1200 Subject: [PATCH 06/30] README -> README.mkd --- README => README.mkd | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename README => README.mkd (100%) diff --git a/README b/README.mkd similarity index 100% rename from README rename to README.mkd From a13fe94b7c97f67731230539264e1d054673d551 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Aug 2015 20:42:52 +1200 Subject: [PATCH 07/30] Cleanup, restructuring, extract version in project config --- rtool | 100 ++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 56 insertions(+), 44 deletions(-) diff --git a/rtool b/rtool index bb33604b0..3c47fc6cc 100755 --- a/rtool +++ b/rtool @@ -11,6 +11,8 @@ import subprocess import glob import re import shlex +import runpy + import click # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes @@ -21,7 +23,6 @@ else: VENV_BIN = "bin" - RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) DIST_DIR = join(RELEASE_DIR, "release") ROOT_DIR = join(RELEASE_DIR, "..") @@ -38,19 +39,36 @@ VENV_DIR = join(RELEASE_DIR, "venv") VENV_PIP = join(VENV_DIR, VENV_BIN, "pip") VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") -PROJECTS = ("netlib", "pathod", "mitmproxy") -TOOLS = { - "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathod", "pathoc"], - "netlib": [] +PROJECTS = { + "netlib": { + "tools": [], + "vfile": join(ROOT_DIR, "netlib/netlib/version.py") + }, + "pathod": { + "tools": ["pathod", "pathoc"], + "vfile": join(ROOT_DIR, "pathod/libpathod/version.py") + }, + "mitmproxy": { + "tools": ["mitmproxy", "mitmdump", "mitmweb"], + "vfile": join(ROOT_DIR, "mitmproxy/libmproxy/version.py") + } } if os.name == "nt": - TOOLS["mitmproxy"].remove("mitmproxy") -VERSION_FILES = { - "mitmproxy": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"), - "pathod": join(ROOT_DIR, "pathod/libpathod/version.py"), - "netlib": join(ROOT_DIR, "netlib/netlib/version.py"), -} + PROJECTS["mitmproxy"]["tools"].remove("mitmproxy") + +for project, settings in PROJECTS.items(): + settings["version"] = runpy.run_path(settings["vfile"])["VERSION"] + settings["dir"] = join(ROOT_DIR, project) + + +def proj(spec): + """ + A small helper to iterate over filtered projects. + """ + for k, v in PROJECTS.items(): + if k not in spec: + continue + yield k, v @contextlib.contextmanager @@ -84,16 +102,14 @@ def cli(): @cli.command("contributors") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS + multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) def contributors(projects): """ Update CONTRIBUTORS.md """ - for project in PROJECTS: - if project not in projects: - continue - with chdir(os.path.join(ROOT_DIR, project)): + for project, conf in proj(projects): + with chdir(conf["dir"]): print("Updating %s/CONTRIBUTORS..."%project) contributors_data = subprocess.check_output( shlex.split("git shortlog -n -s") @@ -118,7 +134,7 @@ def docs(): @cli.command("set-version") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS + multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) @click.argument('version') def set_version(projects, version): @@ -127,24 +143,22 @@ def set_version(projects, version): """ print("Update versions...") version = ", ".join(version.split(".")) - for project, version_file in VERSION_FILES.items(): - if project not in projects: - continue - print("Update %s..." % version_file) - with open(version_file, "rb") as f: + for p, conf in proj(projects): + print("Update %s..." % conf["vfile"]) + with open(conf["vfile"], "rb") as f: content = f.read() new_content = re.sub( r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, content ) - with open(version_file, "wb") as f: + with open(conf["vfile"], "wb") as f: f.write(new_content) @cli.command("git") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS + multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) @click.argument('args', nargs=-1, required=True) def git(projects, args): @@ -152,18 +166,18 @@ def git(projects, args): Run a git command on every project """ args = ["git"] + list(args) - for project in projects: + for project, conf in proj(projects): print("%s> %s..." % (project, " ".join(args))) subprocess.check_call( args, - cwd=join(ROOT_DIR, project) + cwd=conf["dir"] ) @cli.command("sdist") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS + multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) def sdist(projects): """ @@ -173,21 +187,21 @@ def sdist(projects): print("Building release...") if os.path.exists(DIST_DIR): shutil.rmtree(DIST_DIR) - for project in projects: + for project, conf in proj(projects): print("Creating %s source distribution..." % project) subprocess.check_call( [ "python", "./setup.py", "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar" ], - cwd=join(ROOT_DIR, project) + cwd=conf["dir"] ) @cli.command("osxbin") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS + multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) @click.pass_context def osxbin(ctx, projects): @@ -203,8 +217,8 @@ def osxbin(ctx, projects): shutil.rmtree(PYINSTALLER_CACHE, ignore_errors=True) shutil.rmtree("./build", ignore_errors=True) shutil.rmtree(PYINSTALLER_DIST, ignore_errors=True) - for p in projects: - specs = glob.glob(os.path.join(ROOT_DIR, p, "release/*.spec")) + for p, conf in proj(projects): + specs = glob.glob(os.path.join(conf["dir"], "release/*.spec")) for spec in specs: subprocess.check_call( [ @@ -225,12 +239,10 @@ def osxbin(ctx, projects): - - @cli.command("mkvenv") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS + multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) @click.pass_context def mkvenv(ctx, projects): @@ -245,14 +257,13 @@ def mkvenv(ctx, projects): subprocess.check_call(["virtualenv", "-q", VENV_DIR]) with chdir(DIST_DIR): - for project in projects: + for project, conf in proj(projects): print("Installing %s..." % project) - dist = join(ROOT_DIR, project) - subprocess.check_call([VENV_PIP, "install", "-q", dist]) + subprocess.check_call([VENV_PIP, "install", "-q", conf["dir"]]) print("Running binaries...") - for project in projects: - for tool in TOOLS[project]: + for project, conf in proj(projects): + for tool in PROJECTS[project]["tools"]: tool = join(VENV_DIR, VENV_BIN, tool) print(tool) print(subprocess.check_output([tool, "--version"])) @@ -293,16 +304,17 @@ def upload_release(username, password, repository): @click.option('--repository', default="pypi") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS), default=PROJECTS + multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() ) @click.pass_context def wizard(ctx, version, username, password, repository, projects): """ Interactive Release Wizard """ - for project in projects: + for project, conf in proj(projects): if subprocess.check_output( - ["git", "status", "--porcelain"], cwd=join(ROOT_DIR, project) + ["git", "status", "--porcelain"], + cwd=conf["dir"] ): raise RuntimeError("%s repository is not clean." % project) From bf773f822ce948e8d72fdf6676c1fdb6de829cc3 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Aug 2015 21:45:45 +1200 Subject: [PATCH 08/30] OSX tarfiles, remove obsolete osx-binaries --- osx-binaries | 60 ----------------------------------------------- rtool | 66 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 42 insertions(+), 84 deletions(-) delete mode 100755 osx-binaries diff --git a/osx-binaries b/osx-binaries deleted file mode 100755 index 2d1acccc7..000000000 --- a/osx-binaries +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh - -# Quick and dangerous script for building OSX binaries. - -# At the moment, pyinstaller has no support for entry points, except for this -# hideous hack on the wiki: -# https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Setuptools-Entry-Point -# Once this is fixed, we can ditch the redundant command scripts. - -VENV=./venv -PYINST_CMD="$VENV/bin/pyinstaller -F --clean" -TMPDIR=./tmp -CACHE="~/Library/Application Support/pyinstaller" -MITMPROXY="../mitmproxy" - - -if [ ! -d $VENV ] -then - echo "Failed: set up a dev environment as described in the README" - echo "and run from the top-level mitmproxy directory." - exit -fi - -source $VENV/bin/activate - -if [ ! -f $VENV/bin/pyinstaller ] - then - echo "Installing pyinstaller..." - $VENV/bin/pip install \ - --force-reinstall \ - --upgrade \ - https://github.com/pyinstaller/pyinstaller/archive/develop.zip - $VENV/bin/pip install --upgrade macholib -fi - -echo "Clearing caches..." -rm -f dist/* -rm -rf $TMPDIR -rm -rf $CACHE - -# $PYINST_CMD $MITMPROXY/release/mitmdump.spec -# echo "Running mitmdump..." -# ./dist/mitmdump --version || exit 1 - -$PYINST_CMD $MITMPROXY/release/mitmproxy.spec -echo "Running mitmproxy..." -./dist/mitmproxy --version || exit 1 - -# $PYINST_CMD $MITMPROXY/release/mitmweb.spec -# echo "Running mitmweb..." -# ./dist/mitmweb --version || exit 1 - -# DST=osx-mitmproxy-`./dist/mitmdump --shortversion 2>&1` -# mkdir -p $TMPDIR/$DST -# cp ./dist/mitmproxy $TMPDIR/$DST -# cp ./dist/mitmdump $TMPDIR/$DST -# cshape ./doc-src $TMPDIR/$DST/doc -# -# cd $TMPDIR -# tar -czvf $DST.tar.gz $DST diff --git a/rtool b/rtool index 3c47fc6cc..037dcf439 100755 --- a/rtool +++ b/rtool @@ -12,6 +12,7 @@ import glob import re import shlex import runpy +import pprint import click @@ -42,15 +43,21 @@ VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") PROJECTS = { "netlib": { "tools": [], - "vfile": join(ROOT_DIR, "netlib/netlib/version.py") + "vfile": join(ROOT_DIR, "netlib/netlib/version.py"), + "version": None, + "dir": None }, "pathod": { "tools": ["pathod", "pathoc"], - "vfile": join(ROOT_DIR, "pathod/libpathod/version.py") + "vfile": join(ROOT_DIR, "pathod/libpathod/version.py"), + "version": None, + "dir": None }, "mitmproxy": { "tools": ["mitmproxy", "mitmdump", "mitmweb"], - "vfile": join(ROOT_DIR, "mitmproxy/libmproxy/version.py") + "vfile": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"), + "version": None, + "dir": None } } if os.name == "nt": @@ -201,42 +208,53 @@ def sdist(projects): @cli.command("osxbin") @click.option( '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() + multiple=True, + type=click.Choice(PROJECTS.keys()), + default=PROJECTS.keys() ) @click.pass_context def osxbin(ctx, projects): if not os.path.exists(VENV_PYINSTALLER): - subprocess.check_call( - [ - VENV_PIP, - "install", - PYINSTALLER_URL - ] - ) + print("Instaling PyInstaller...") + subprocess.check_call( + [ + VENV_PIP, + "install", + PYINSTALLER_URL + ] + ) shutil.rmtree(PYINSTALLER_CACHE, ignore_errors=True) shutil.rmtree("./build", ignore_errors=True) shutil.rmtree(PYINSTALLER_DIST, ignore_errors=True) for p, conf in proj(projects): specs = glob.glob(os.path.join(conf["dir"], "release/*.spec")) - for spec in specs: - subprocess.check_call( - [ - VENV_PYINSTALLER, - "--distpath", PYINSTALLER_DIST, - spec - ] - ) - if specs and os.path.exists(PYINSTALLER_DIST): - bins = os.listdir(PYINSTALLER_DIST) - for bin in bins: + if specs: + for spec in specs: subprocess.check_call( [ - os.path.join(PYINSTALLER_DIST, bin), - "--version" + VENV_PYINSTALLER, + "--distpath", PYINSTALLER_DIST, + spec ] ) + bins = os.listdir(PYINSTALLER_DIST) + base = os.path.join(DIST_DIR, "osx-" + p + "-" + conf["version"]) + shutil.rmtree(base, ignore_errors=True) + os.makedirs(base) + for bin in bins: + bin = os.path.join(PYINSTALLER_DIST, bin) + subprocess.check_call([bin, "--version"]) + shutil.move(bin, base) + subprocess.check_call( + [ + "tar", + "-czvf", + base + ".tgz", + base + ] + ) @cli.command("mkvenv") From 178324a0eea184930a3f7eb58cd85861f1acd97e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 27 Nov 2015 20:26:02 +0100 Subject: [PATCH 09/30] remove unicode_literals --- rtool | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtool b/rtool index 037dcf439..7a4b0d58e 100755 --- a/rtool +++ b/rtool @@ -1,6 +1,6 @@ #!/usr/bin/env python from __future__ import ( - absolute_import, print_function, division, unicode_literals + absolute_import, print_function, division ) from os.path import join import contextlib From decdb75fbaccbcd041433542f10163c104c43b9c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 29 Nov 2015 02:46:08 +0100 Subject: [PATCH 10/30] update rtool --- .env | 7 +- .gitignore | 8 +- requirements.txt | 2 - rtool | 368 ----------------------------------------------- rtool.py | 315 ++++++++++++++++++++++++++++++++++++++++ setup.py | 16 +++ 6 files changed, 337 insertions(+), 379 deletions(-) delete mode 100644 requirements.txt delete mode 100755 rtool create mode 100644 rtool.py create mode 100644 setup.py diff --git a/.env b/.env index 97f38452b..69ac3f052 100644 --- a/.env +++ b/.env @@ -1,5 +1,6 @@ -DIR="${0%/*}" -if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then +DIR="$( dirname "${BASH_SOURCE[0]}" )" +ACTIVATE_DIR="$(if [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then echo 'bin'; else echo 'Scripts'; fi;)" +if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/$ACTIVATE_DIR/activate" ]; then echo "Activating mitmproxy virtualenv..." - source "$DIR/../venv.mitmproxy/bin/activate" + source "$DIR/../venv.mitmproxy/$ACTIVATE_DIR/activate" fi diff --git a/.gitignore b/.gitignore index b3f4fc3e0..c062fb3b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,9 @@ .DS_Store MANIFEST -/dist -/tmp -/venv *.py[cdo] *.swp *.swo -/venv -/release /build -/pyinstallerdist +/dist +/mitmproxy_rtool.egg-info \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index de1b90b2a..000000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -click>=4.1 -twine>=1.5.0 diff --git a/rtool b/rtool deleted file mode 100755 index 7a4b0d58e..000000000 --- a/rtool +++ /dev/null @@ -1,368 +0,0 @@ -#!/usr/bin/env python -from __future__ import ( - absolute_import, print_function, division -) -from os.path import join -import contextlib -import os.path -import os -import shutil -import subprocess -import glob -import re -import shlex -import runpy -import pprint - -import click - -# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes -# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ -if os.name == "nt": - VENV_BIN = "Scripts" -else: - VENV_BIN = "bin" - - -RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) -DIST_DIR = join(RELEASE_DIR, "release") -ROOT_DIR = join(RELEASE_DIR, "..") -MITMPROXY_DIR = join(ROOT_DIR, "mitmproxy") - -PYINSTALLER_URL =\ - "https://github.com/pyinstaller/pyinstaller/archive/develop.zip" -PYINSTALLER_CACHE = os.path.expanduser( - "~/Library/Application Support/pyinstaller" -) -PYINSTALLER_DIST = join(RELEASE_DIR, "pyinstallerdist") - -VENV_DIR = join(RELEASE_DIR, "venv") -VENV_PIP = join(VENV_DIR, VENV_BIN, "pip") -VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") - -PROJECTS = { - "netlib": { - "tools": [], - "vfile": join(ROOT_DIR, "netlib/netlib/version.py"), - "version": None, - "dir": None - }, - "pathod": { - "tools": ["pathod", "pathoc"], - "vfile": join(ROOT_DIR, "pathod/libpathod/version.py"), - "version": None, - "dir": None - }, - "mitmproxy": { - "tools": ["mitmproxy", "mitmdump", "mitmweb"], - "vfile": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"), - "version": None, - "dir": None - } -} -if os.name == "nt": - PROJECTS["mitmproxy"]["tools"].remove("mitmproxy") - -for project, settings in PROJECTS.items(): - settings["version"] = runpy.run_path(settings["vfile"])["VERSION"] - settings["dir"] = join(ROOT_DIR, project) - - -def proj(spec): - """ - A small helper to iterate over filtered projects. - """ - for k, v in PROJECTS.items(): - if k not in spec: - continue - yield k, v - - -@contextlib.contextmanager -def empty_pythonpath(): - """ - Make sure that the regular python installation is not on the python path, - which would give us access to modules installed outside of our virtualenv. - """ - pythonpath = os.environ["PYTHONPATH"] - os.environ["PYTHONPATH"] = "" - yield - os.environ["PYTHONPATH"] = pythonpath - - -@contextlib.contextmanager -def chdir(path): - old_dir = os.getcwd() - os.chdir(path) - yield - os.chdir(old_dir) - - -@click.group(chain=True) -def cli(): - """ - mitmproxy build tool - """ - pass - - -@cli.command("contributors") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() -) -def contributors(projects): - """ - Update CONTRIBUTORS.md - """ - for project, conf in proj(projects): - with chdir(conf["dir"]): - print("Updating %s/CONTRIBUTORS..."%project) - contributors_data = subprocess.check_output( - shlex.split("git shortlog -n -s") - ) - with open("CONTRIBUTORS", "w+") as f: - f.write(contributors_data) - - -@cli.command("docs") -def docs(): - """ - Render the docs - """ - print("Rendering the docs...") - subprocess.check_call([ - "cshape", - join(MITMPROXY_DIR, "doc-src"), - join(MITMPROXY_DIR, "doc") - ]) - - -@cli.command("set-version") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() -) -@click.argument('version') -def set_version(projects, version): - """ - Update version information - """ - print("Update versions...") - version = ", ".join(version.split(".")) - for p, conf in proj(projects): - print("Update %s..." % conf["vfile"]) - with open(conf["vfile"], "rb") as f: - content = f.read() - new_content = re.sub( - r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, - content - ) - with open(conf["vfile"], "wb") as f: - f.write(new_content) - - -@cli.command("git") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() -) -@click.argument('args', nargs=-1, required=True) -def git(projects, args): - """ - Run a git command on every project - """ - args = ["git"] + list(args) - for project, conf in proj(projects): - print("%s> %s..." % (project, " ".join(args))) - subprocess.check_call( - args, - cwd=conf["dir"] - ) - - -@cli.command("sdist") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() -) -def sdist(projects): - """ - Build a source distribution - """ - with empty_pythonpath(): - print("Building release...") - if os.path.exists(DIST_DIR): - shutil.rmtree(DIST_DIR) - for project, conf in proj(projects): - print("Creating %s source distribution..." % project) - subprocess.check_call( - [ - "python", "./setup.py", - "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar" - ], - cwd=conf["dir"] - ) - - -@cli.command("osxbin") -@click.option( - '--project', '-p', 'projects', - multiple=True, - type=click.Choice(PROJECTS.keys()), - default=PROJECTS.keys() -) -@click.pass_context -def osxbin(ctx, projects): - if not os.path.exists(VENV_PYINSTALLER): - print("Instaling PyInstaller...") - subprocess.check_call( - [ - VENV_PIP, - "install", - PYINSTALLER_URL - ] - ) - - shutil.rmtree(PYINSTALLER_CACHE, ignore_errors=True) - shutil.rmtree("./build", ignore_errors=True) - shutil.rmtree(PYINSTALLER_DIST, ignore_errors=True) - for p, conf in proj(projects): - specs = glob.glob(os.path.join(conf["dir"], "release/*.spec")) - if specs: - for spec in specs: - subprocess.check_call( - [ - VENV_PYINSTALLER, - "--distpath", PYINSTALLER_DIST, - spec - ] - ) - bins = os.listdir(PYINSTALLER_DIST) - - base = os.path.join(DIST_DIR, "osx-" + p + "-" + conf["version"]) - shutil.rmtree(base, ignore_errors=True) - os.makedirs(base) - for bin in bins: - bin = os.path.join(PYINSTALLER_DIST, bin) - subprocess.check_call([bin, "--version"]) - shutil.move(bin, base) - subprocess.check_call( - [ - "tar", - "-czvf", - base + ".tgz", - base - ] - ) - - -@cli.command("mkvenv") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() -) -@click.pass_context -def mkvenv(ctx, projects): - """ - make a venv and test the source distribution - """ - ctx.invoke(sdist) - with empty_pythonpath(): - print("Creating virtualenv for test install...") - if os.path.exists(VENV_DIR): - shutil.rmtree(VENV_DIR) - subprocess.check_call(["virtualenv", "-q", VENV_DIR]) - - with chdir(DIST_DIR): - for project, conf in proj(projects): - print("Installing %s..." % project) - subprocess.check_call([VENV_PIP, "install", "-q", conf["dir"]]) - - print("Running binaries...") - for project, conf in proj(projects): - for tool in PROJECTS[project]["tools"]: - tool = join(VENV_DIR, VENV_BIN, tool) - print(tool) - print(subprocess.check_output([tool, "--version"])) - - print("Virtualenv available for further testing:") - print( - "source %s" % os.path.normpath( - join(VENV_DIR, VENV_BIN, "activate") - ) - ) - - -@cli.command("upload") -@click.option('--username', prompt=True) -@click.password_option(confirmation_prompt=False) -@click.option('--repository', default="pypi") -def upload_release(username, password, repository): - """ - Upload source distributions to PyPI - """ - print("Uploading distributions...") - subprocess.check_call([ - "twine", - "upload", - "-u", username, - "-p", password, - "-r", repository, - "%s/*" % DIST_DIR - ]) - - -# TODO: Fully automate build process. -# This wizard is missing OSX builds and updating mitmproxy.org. -@cli.command("wizard") -@click.option('--version', prompt=True) -@click.option('--username', prompt="PyPI Username") -@click.password_option(confirmation_prompt=False, prompt="PyPI Password") -@click.option('--repository', default="pypi") -@click.option( - '--project', '-p', 'projects', - multiple=True, type=click.Choice(PROJECTS.keys()), default=PROJECTS.keys() -) -@click.pass_context -def wizard(ctx, version, username, password, repository, projects): - """ - Interactive Release Wizard - """ - for project, conf in proj(projects): - if subprocess.check_output( - ["git", "status", "--porcelain"], - cwd=conf["dir"] - ): - raise RuntimeError("%s repository is not clean." % project) - - # Build test release - ctx.invoke(sdist, projects=projects) - ctx.invoke(test, projects=projects) - click.confirm("Please test the release now. Is it ok?", abort=True) - - # bump version, update docs and contributors - ctx.invoke(set_version, version=version, projects=projects) - ctx.invoke(docs) - ctx.invoke(contributors) - - # version bump commit + tag - ctx.invoke( - git, args=["commit", "-a", "-m", "bump version"], projects=projects - ) - ctx.invoke(git, args=["tag", "v" + version], projects=projects) - ctx.invoke(git, args=["push"], projects=projects) - ctx.invoke(git, args=["push", "--tags"], projects=projects) - - # Re-invoke sdist with bumped version - ctx.invoke(sdist, projects=projects) - click.confirm("All good, can upload to PyPI?", abort=True) - ctx.invoke( - upload_release, - username=username, password=password, repository=repository - ) - click.echo("All done!") - - -if __name__ == "__main__": - cli() diff --git a/rtool.py b/rtool.py new file mode 100644 index 000000000..c9e697c6d --- /dev/null +++ b/rtool.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python +from __future__ import absolute_import, print_function, division + +from os.path import join +import contextlib +import os +import shutil +import subprocess +import glob +import re +import shlex +import runpy +import pprint +from zipfile import ZipFile +from tarfile import TarFile +import platform + +import click + +# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes +# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ +if platform.system() == "Windows": + VENV_BIN = "Scripts" +else: + VENV_BIN = "bin" + +if platform.system() == "Windows": + def Archive(name): + a = ZipFile(name + ".zip","w") + a.add = a.write + return a +else: + def Archive(name): + a = TarFile(name + ".tar.gz", "w:gz") + return a + + +RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) +DIST_DIR = join(RELEASE_DIR, "dist") +ROOT_DIR = join(RELEASE_DIR, "..") + +BUILD_DIR = join(RELEASE_DIR, "build") +PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller") +PYINSTALLER_DIST = join(BUILD_DIR, "binaries") + +VENV_DIR = join(BUILD_DIR, "venv") +VENV_PIP = join(VENV_DIR, VENV_BIN, "pip") +VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") + + +ALL_PROJECTS = { + "netlib": { + "tools": [], + "vfile": join(ROOT_DIR, "netlib/netlib/version.py"), + "dir": join(ROOT_DIR, "netlib") + }, + "pathod": { + "tools": ["pathod", "pathoc"], + "vfile": join(ROOT_DIR, "pathod/libpathod/version.py"), + "dir": join(ROOT_DIR, "pathod") + }, + "mitmproxy": { + "tools": ["mitmproxy", "mitmdump", "mitmweb"], + "vfile": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"), + "dir": join(ROOT_DIR, "mitmproxy") + } +} +if platform.system() == "Windows": + ALL_PROJECTS["mitmproxy"]["tools"].remove("mitmproxy") + +projects = {} + +def version(project): + return runpy.run_path(projects[project]["vfile"])["VERSION"] + +def sdist_name(project): + return "{project}-{version}.tar.gz".format(project=project, version=version(project)) + +@contextlib.contextmanager +def empty_pythonpath(): + """ + Make sure that the regular python installation is not on the python path, + which would give us access to modules installed outside of our virtualenv. + """ + pythonpath = os.environ.get("PYTHONPATH","") + os.environ["PYTHONPATH"] = "" + yield + os.environ["PYTHONPATH"] = pythonpath + + +@contextlib.contextmanager +def chdir(path): + old_dir = os.getcwd() + os.chdir(path) + yield + os.chdir(old_dir) + + +@click.group(chain=True) +@click.option( + '--project', '-p', + multiple=True, type=click.Choice(ALL_PROJECTS.keys()), default=ALL_PROJECTS.keys() +) +def cli(project): + """ + mitmproxy build tool + """ + for name in project: + projects[name] = ALL_PROJECTS[name] + +@cli.command("contributors") +def contributors(): + """ + Update CONTRIBUTORS.md + """ + for project, conf in projects.items(): + with chdir(conf["dir"]): + print("Updating %s/CONTRIBUTORS..." % project) + contributors_data = subprocess.check_output( + shlex.split("git shortlog -n -s") + ) + with open("CONTRIBUTORS", "w") as f: + f.write(contributors_data) + +@cli.command("set-version") +@click.argument('version') +def set_version(version): + """ + Update version information + """ + print("Update versions...") + version = ", ".join(version.split(".")) + for p, conf in projects.items(): + print("Update %s..." % os.path.normpath(conf["vfile"])) + with open(conf["vfile"], "rb") as f: + content = f.read() + new_content = re.sub( + r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version, + content + ) + with open(conf["vfile"], "wb") as f: + f.write(new_content) + + +@cli.command("git") +@click.argument('args', nargs=-1, required=True) +def git(args): + """ + Run a git command on every project + """ + args = ["git"] + list(args) + for project, conf in projects.items(): + print("%s> %s..." % (project, " ".join(args))) + subprocess.check_call( + args, + cwd=conf["dir"] + ) + + +@cli.command("sdist") +def sdist(): + """ + Build a source distribution + """ + with empty_pythonpath(): + print("Building release...") + if os.path.exists(DIST_DIR): + shutil.rmtree(DIST_DIR) + for project, conf in projects.items(): + print("Creating %s source distribution..." % project) + subprocess.check_call( + [ + "python", "./setup.py", + "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar" + ], + cwd=conf["dir"] + ) + + print("Creating virtualenv for test install...") + if os.path.exists(VENV_DIR): + shutil.rmtree(VENV_DIR) + subprocess.check_call(["virtualenv", "-q", VENV_DIR]) + + with chdir(DIST_DIR): + for project, conf in projects.items(): + print("Installing %s..." % project) + subprocess.check_call([VENV_PIP, "install", "-q", sdist_name(project)]) + + print("Running binaries...") + for project, conf in projects.items(): + for tool in conf["tools"]: + tool = join(VENV_DIR, VENV_BIN, tool) + print("> %s --version" % tool) + print(subprocess.check_output([tool, "--version"])) + + print("Virtualenv available for further testing:") + print("source %s" % os.path.normpath(join(VENV_DIR, VENV_BIN, "activate"))) + + +@cli.command("bdist") +@click.option('--use-existing-sdist/--no-use-existing-sdist', default=False) +@click.pass_context +def bdist(ctx, use_existing_sdist): + """ + Build a binary distribution + """ + if os.path.exists(PYINSTALLER_TEMP): + shutil.rmtree(PYINSTALLER_TEMP) + if os.path.exists(PYINSTALLER_DIST): + shutil.rmtree(PYINSTALLER_DIST) + + if not use_existing_sdist: + ctx.invoke(sdist) + + print("Installing PyInstaller...") + subprocess.check_call([VENV_PIP, "install", "-q", "PyInstaller~=3.0.0"]) + + for p, conf in projects.items(): + if conf["tools"]: + archive_name = "{project}-{version}-{platform}".format( + project=p, + version=version(p), + platform=platform.system() + ) + with Archive(join(DIST_DIR, archive_name)) as archive: + for tool in conf["tools"]: + spec = join(conf["dir"], "release", "%s.spec" % tool) + print("Building %s binary..." % tool) + subprocess.check_call( + [ + VENV_PYINSTALLER, + "--clean", + "--workpath", PYINSTALLER_TEMP, + "--distpath", PYINSTALLER_DIST, + # This is PyInstaller, so setting a + # different log level obviously breaks it :-) + # "--log-level", "WARN", + spec + ] + ) + + # Test if it works at all O:-) + executable = join(PYINSTALLER_DIST, tool) + if platform.system() == "Windows": + executable += ".exe" + print("Testinng %s..." % executable) + subprocess.check_call([executable, "--version"]) + + archive.add(executable, os.path.basename(executable)) + + +@cli.command("upload") +@click.option('--username', prompt=True) +@click.password_option(confirmation_prompt=False) +@click.option('--repository', default="pypi") +def upload_release(username, password, repository): + """ + Upload source distributions to PyPI + """ + + for project in projects.keys(): + print("Uploading {} to {}...".format(project, repository)) + subprocess.check_call([ + "twine", + "upload", + "-u", username, + "-p", password, + "-r", repository, + join(DIST_DIR, sdist_name(project)) + ]) + + +@cli.command("wizard") +@click.option('--version', prompt=True) +@click.option('--username', prompt="PyPI Username") +@click.password_option(confirmation_prompt=False, prompt="PyPI Password") +@click.option('--repository', default="pypi") +@click.pass_context +def wizard(ctx, version, username, password, repository): + """ + Interactive Release Wizard + """ + for project, conf in projects.items(): + is_dirty = subprocess.check_output(["git", "status", "--porcelain"], cwd=conf["dir"]) + if is_dirty: + raise RuntimeError("%s repository is not clean." % project) + + # Build test release + ctx.invoke(bdist) + click.confirm("Please test the release now. Is it ok?", abort=True) + + # bump version, update docs and contributors + ctx.invoke(set_version, version=version) + ctx.invoke(contributors) + + # version bump commit + tag + ctx.invoke( + git, args=["commit", "-a", "-m", "bump version"] + ) + ctx.invoke(git, args=["tag", version]) + ctx.invoke(git, args=["push"]) + ctx.invoke(git, args=["push", "--tags"]) + + # Re-invoke sdist with bumped version + ctx.invoke(sdist) + click.confirm("All good, can upload sdist to PyPI?", abort=True) + ctx.invoke( + upload_release, + username=username, password=password, repository=repository + ) + click.echo("All done!") + + +if __name__ == "__main__": + cli() diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..06ac230db --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup, find_packages + +setup( + name='mitmproxy-rtool', + version='1.0', + py_modules=['rtool'], + install_requires=[ + 'click~=6.2', + 'twine~=1.6.4', + ], + entry_points={ + 'console_scripts': [ + 'rtool=rtool:cli', + ], + }, +) From 45c199a80720f7e615b70919d592be31226e9734 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 29 Nov 2015 03:38:23 +0100 Subject: [PATCH 11/30] fix tarfile creation --- rtool.py | 5 ++--- test-release | 38 -------------------------------------- 2 files changed, 2 insertions(+), 41 deletions(-) delete mode 100755 test-release diff --git a/rtool.py b/rtool.py index c9e697c6d..e12592194 100644 --- a/rtool.py +++ b/rtool.py @@ -12,7 +12,7 @@ import shlex import runpy import pprint from zipfile import ZipFile -from tarfile import TarFile +import tarfile import platform import click @@ -31,8 +31,7 @@ if platform.system() == "Windows": return a else: def Archive(name): - a = TarFile(name + ".tar.gz", "w:gz") - return a + return tarfile.open(name + ".tar.gz", "w:gz") RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) diff --git a/test-release b/test-release deleted file mode 100755 index 8cbcea8c5..000000000 --- a/test-release +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -MITMPROXY_DIR=~/mitmproxy/mitmproxy -NETLIB_DIR=~/mitmproxy/netlib -PATHOD_DIR=~/mitmproxy/pathod -DST=/tmp/mitmproxy_release - -rm -rf $DST -mkdir -p $DST - -cd $NETLIB_DIR -echo "Creating netlib source distribution..." -python ./setup.py -q sdist --dist-dir $DST - -echo "Creating mitmproxy source distribution..." -cd $MITMPROXY_DIR -python ./setup.py -q sdist --dist-dir $DST - -echo "Creating pathod source distribution..." -cd $PATHOD_DIR -python ./setup.py -q sdist --dist-dir $DST - -echo "Creating virtualenv for test install..." -virtualenv -q $DST/venv - -cd $DST -echo "Installing netlib..." -./venv/bin/pip -q install --download-cache ~/.pipcache ./netlib* -echo "Installing pathod..." -./venv/bin/pip -q install --download-cache ~/.pipcache ./pathod* -echo "Installing mitmproxy..." -./venv/bin/pip -q install --download-cache ~/.pipcache ./mitmproxy* - -echo "Running binaries..." -./venv/bin/mitmproxy --version -./venv/bin/mitmdump --version -./venv/bin/pathod --version -./venv/bin/pathoc --version From 0188d5b1c7ffe93bbc7947ebf995a955e590ee6e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 29 Nov 2015 14:50:44 +0100 Subject: [PATCH 12/30] better binary archive names --- rtool.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rtool.py b/rtool.py index e12592194..73a43f9a4 100644 --- a/rtool.py +++ b/rtool.py @@ -219,7 +219,11 @@ def bdist(ctx, use_existing_sdist): archive_name = "{project}-{version}-{platform}".format( project=p, version=version(p), - platform=platform.system() + platform={ + "Darwin": "osx", + "Windows": "win32", + "Linux": "linux" + }.get(platform.system(), platform.system()) ) with Archive(join(DIST_DIR, archive_name)) as archive: for tool in conf["tools"]: From 8cd9f419294a5b192078924ed008e39bff1674db Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 29 Nov 2015 14:54:44 +0100 Subject: [PATCH 13/30] tabs -> spaces --- rtool.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/rtool.py b/rtool.py index 73a43f9a4..822443816 100644 --- a/rtool.py +++ b/rtool.py @@ -73,7 +73,7 @@ def version(project): return runpy.run_path(projects[project]["vfile"])["VERSION"] def sdist_name(project): - return "{project}-{version}.tar.gz".format(project=project, version=version(project)) + return "{project}-{version}.tar.gz".format(project=project, version=version(project)) @contextlib.contextmanager def empty_pythonpath(): @@ -222,7 +222,7 @@ def bdist(ctx, use_existing_sdist): platform={ "Darwin": "osx", "Windows": "win32", - "Linux": "linux" + "Linux": "linux" }.get(platform.system(), platform.system()) ) with Archive(join(DIST_DIR, archive_name)) as archive: @@ -262,15 +262,15 @@ def upload_release(username, password, repository): """ for project in projects.keys(): - print("Uploading {} to {}...".format(project, repository)) - subprocess.check_call([ - "twine", - "upload", - "-u", username, - "-p", password, - "-r", repository, - join(DIST_DIR, sdist_name(project)) - ]) + print("Uploading {} to {}...".format(project, repository)) + subprocess.check_call([ + "twine", + "upload", + "-u", username, + "-p", password, + "-r", repository, + join(DIST_DIR, sdist_name(project)) + ]) @cli.command("wizard") From 459ab0434fb37ae6ec8777c0118fa13033fb82d7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 29 Nov 2015 15:25:12 +0100 Subject: [PATCH 14/30] add virtualenv dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 06ac230db..2422d7a93 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ setup( install_requires=[ 'click~=6.2', 'twine~=1.6.4', + 'virtualenv~=13.1.2', ], entry_points={ 'console_scripts': [ From d3570747e8ef4ba4068edd89501504df4ce3281c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 29 Nov 2015 19:05:58 +0100 Subject: [PATCH 15/30] also build wheels --- rtool.py | 36 ++++++++++++++++++++++++------------ setup.py | 7 ++++--- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/rtool.py b/rtool.py index 822443816..f901fb648 100644 --- a/rtool.py +++ b/rtool.py @@ -14,6 +14,7 @@ import pprint from zipfile import ZipFile import tarfile import platform +import sys import click @@ -73,7 +74,17 @@ def version(project): return runpy.run_path(projects[project]["vfile"])["VERSION"] def sdist_name(project): - return "{project}-{version}.tar.gz".format(project=project, version=version(project)) + return "{project}-{version}.tar.gz".format( + project=project, + version=version(project) + ) + +def wheel_name(project): + return "{project}-{version}-py{py_version}-none-any.whl".format( + project=project, + version=version(project), + py_version=sys.version_info.major + ) @contextlib.contextmanager def empty_pythonpath(): @@ -170,7 +181,7 @@ def sdist(): subprocess.check_call( [ "python", "./setup.py", - "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar" + "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar", "bdist_wheel", "--dist-dir", DIST_DIR, ], cwd=conf["dir"] ) @@ -183,7 +194,7 @@ def sdist(): with chdir(DIST_DIR): for project, conf in projects.items(): print("Installing %s..." % project) - subprocess.check_call([VENV_PIP, "install", "-q", sdist_name(project)]) + subprocess.check_call([VENV_PIP, "install", "-q", wheel_name(project)]) print("Running binaries...") for project, conf in projects.items(): @@ -262,15 +273,16 @@ def upload_release(username, password, repository): """ for project in projects.keys(): - print("Uploading {} to {}...".format(project, repository)) - subprocess.check_call([ - "twine", - "upload", - "-u", username, - "-p", password, - "-r", repository, - join(DIST_DIR, sdist_name(project)) - ]) + for f in (sdist_name(project), wheel_name(project)): + print("Uploading {} to {}...".format(f, repository)) + subprocess.check_call([ + "twine", + "upload", + "-u", username, + "-p", password, + "-r", repository, + join(DIST_DIR, f) + ]) @cli.command("wizard") diff --git a/setup.py b/setup.py index 2422d7a93..e99c159cf 100644 --- a/setup.py +++ b/setup.py @@ -5,9 +5,10 @@ setup( version='1.0', py_modules=['rtool'], install_requires=[ - 'click~=6.2', - 'twine~=1.6.4', - 'virtualenv~=13.1.2', + 'click>=6.2', + 'twine>=1.6.4', + 'virtualenv>=13.1.2', + 'wheel>=0.26.0', ], entry_points={ 'console_scripts': [ From a0a7be9ab1c46e94f22b2d2b608769fc11ca0270 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 29 Nov 2015 22:11:14 +0100 Subject: [PATCH 16/30] make pyinstaller version configurable --- rtool.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rtool.py b/rtool.py index f901fb648..d66b26361 100644 --- a/rtool.py +++ b/rtool.py @@ -208,9 +208,10 @@ def sdist(): @cli.command("bdist") -@click.option('--use-existing-sdist/--no-use-existing-sdist', default=False) +@click.option("--use-existing-sdist/--no-use-existing-sdist", default=False) +@click.argument("pyinstaller_version", envvar="PYINSTALLER_VERSION", default="PyInstaller~=3.0.0") @click.pass_context -def bdist(ctx, use_existing_sdist): +def bdist(ctx, use_existing_sdist, pyinstaller_version): """ Build a binary distribution """ @@ -223,7 +224,7 @@ def bdist(ctx, use_existing_sdist): ctx.invoke(sdist) print("Installing PyInstaller...") - subprocess.check_call([VENV_PIP, "install", "-q", "PyInstaller~=3.0.0"]) + subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version]) for p, conf in projects.items(): if conf["tools"]: @@ -261,6 +262,7 @@ def bdist(ctx, use_existing_sdist): subprocess.check_call([executable, "--version"]) archive.add(executable, os.path.basename(executable)) + print("Packed {}.".format(archive_name)) @cli.command("upload") From 7c14725e547aeed668318cdc3f3604143ebb2167 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 3 Dec 2015 14:28:32 +0100 Subject: [PATCH 17/30] add 'v' in release tag --- rtool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtool.py b/rtool.py index d66b26361..8a38f8708 100644 --- a/rtool.py +++ b/rtool.py @@ -314,7 +314,7 @@ def wizard(ctx, version, username, password, repository): ctx.invoke( git, args=["commit", "-a", "-m", "bump version"] ) - ctx.invoke(git, args=["tag", version]) + ctx.invoke(git, args=["tag", "v" + version]) ctx.invoke(git, args=["push"]) ctx.invoke(git, args=["push", "--tags"]) From 79da0f7c93d650d7df22bcd7c521ae3b6f50c461 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 3 Dec 2015 17:55:42 +0100 Subject: [PATCH 18/30] clean up, don't build wheels --- rtool.py | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/rtool.py b/rtool.py index 8a38f8708..a7dda5c8d 100644 --- a/rtool.py +++ b/rtool.py @@ -1,21 +1,17 @@ #!/usr/bin/env python from __future__ import absolute_import, print_function, division - from os.path import join import contextlib import os import shutil import subprocess -import glob import re import shlex import runpy -import pprint -from zipfile import ZipFile +import zipfile import tarfile import platform import sys - import click # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes @@ -27,14 +23,13 @@ else: if platform.system() == "Windows": def Archive(name): - a = ZipFile(name + ".zip","w") + a = zipfile.ZipFile(name + ".zip", "w") a.add = a.write return a else: def Archive(name): return tarfile.open(name + ".tar.gz", "w:gz") - RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) DIST_DIR = join(RELEASE_DIR, "dist") ROOT_DIR = join(RELEASE_DIR, "..") @@ -47,7 +42,6 @@ VENV_DIR = join(BUILD_DIR, "venv") VENV_PIP = join(VENV_DIR, VENV_BIN, "pip") VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller") - ALL_PROJECTS = { "netlib": { "tools": [], @@ -70,29 +64,33 @@ if platform.system() == "Windows": projects = {} -def version(project): + +def get_version(project): return runpy.run_path(projects[project]["vfile"])["VERSION"] + def sdist_name(project): return "{project}-{version}.tar.gz".format( project=project, - version=version(project) + version=get_version(project) ) + def wheel_name(project): return "{project}-{version}-py{py_version}-none-any.whl".format( project=project, - version=version(project), + version=get_version(project), py_version=sys.version_info.major ) + @contextlib.contextmanager def empty_pythonpath(): """ Make sure that the regular python installation is not on the python path, which would give us access to modules installed outside of our virtualenv. """ - pythonpath = os.environ.get("PYTHONPATH","") + pythonpath = os.environ.get("PYTHONPATH", "") os.environ["PYTHONPATH"] = "" yield os.environ["PYTHONPATH"] = pythonpath @@ -118,6 +116,7 @@ def cli(project): for name in project: projects[name] = ALL_PROJECTS[name] + @cli.command("contributors") def contributors(): """ @@ -132,6 +131,7 @@ def contributors(): with open("CONTRIBUTORS", "w") as f: f.write(contributors_data) + @cli.command("set-version") @click.argument('version') def set_version(version): @@ -180,8 +180,12 @@ def sdist(): print("Creating %s source distribution..." % project) subprocess.check_call( [ - "python", "./setup.py", - "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar", "bdist_wheel", "--dist-dir", DIST_DIR, + "python", "./setup.py", "-q", + "sdist", "--dist-dir", DIST_DIR, "--formats=gztar", + # We are currently not building wheels: mitmproxy has dependencies varying by platform, + # so we'd need environment markers, which are not supported by old versions of setuptools. + # Providing just the source is a bit slower, but it works everywhere. + # "bdist_wheel", "--dist-dir", DIST_DIR, ], cwd=conf["dir"] ) @@ -229,8 +233,8 @@ def bdist(ctx, use_existing_sdist, pyinstaller_version): for p, conf in projects.items(): if conf["tools"]: archive_name = "{project}-{version}-{platform}".format( - project=p, - version=version(p), + project=p, + version=get_version(p), platform={ "Darwin": "osx", "Windows": "win32", @@ -273,7 +277,6 @@ def upload_release(username, password, repository): """ Upload source distributions to PyPI """ - for project in projects.keys(): for f in (sdist_name(project), wheel_name(project)): print("Uploading {} to {}...".format(f, repository)) @@ -302,14 +305,14 @@ def wizard(ctx, version, username, password, repository): if is_dirty: raise RuntimeError("%s repository is not clean." % project) - # Build test release - ctx.invoke(bdist) - click.confirm("Please test the release now. Is it ok?", abort=True) - # bump version, update docs and contributors ctx.invoke(set_version, version=version) ctx.invoke(contributors) + # Build test release + ctx.invoke(bdist) + click.confirm("Please test the release now. Is it ok?", abort=True) + # version bump commit + tag ctx.invoke( git, args=["commit", "-a", "-m", "bump version"] From d6c0157f7d11912c9d5f81e5086603054516635e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 3 Dec 2015 18:03:59 +0100 Subject: [PATCH 19/30] fully disable wheels --- rtool.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rtool.py b/rtool.py index a7dda5c8d..95753b6a3 100644 --- a/rtool.py +++ b/rtool.py @@ -198,7 +198,7 @@ def sdist(): with chdir(DIST_DIR): for project, conf in projects.items(): print("Installing %s..." % project) - subprocess.check_call([VENV_PIP, "install", "-q", wheel_name(project)]) + subprocess.check_call([VENV_PIP, "install", "-q", sdist_name(project)]) print("Running binaries...") for project, conf in projects.items(): @@ -278,7 +278,12 @@ def upload_release(username, password, repository): Upload source distributions to PyPI """ for project in projects.keys(): - for f in (sdist_name(project), wheel_name(project)): + files = ( + sdist_name(project), + # See sdist why this is disabled. + # wheel_name(project) + ) + for f in files: print("Uploading {} to {}...".format(f, repository)) subprocess.check_call([ "twine", From 9554ce9ab43a209777936f51b6fbef1ac2ddd277 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 4 Feb 2016 23:03:13 +0100 Subject: [PATCH 20/30] re-enable wheels --- rtool.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/rtool.py b/rtool.py index 95753b6a3..78cf7bd9f 100644 --- a/rtool.py +++ b/rtool.py @@ -182,10 +182,7 @@ def sdist(): [ "python", "./setup.py", "-q", "sdist", "--dist-dir", DIST_DIR, "--formats=gztar", - # We are currently not building wheels: mitmproxy has dependencies varying by platform, - # so we'd need environment markers, which are not supported by old versions of setuptools. - # Providing just the source is a bit slower, but it works everywhere. - # "bdist_wheel", "--dist-dir", DIST_DIR, + "bdist_wheel", "--dist-dir", DIST_DIR, ], cwd=conf["dir"] ) @@ -280,8 +277,7 @@ def upload_release(username, password, repository): for project in projects.keys(): files = ( sdist_name(project), - # See sdist why this is disabled. - # wheel_name(project) + wheel_name(project) ) for f in files: print("Uploading {} to {}...".format(f, repository)) From 4a6db7671f2a472e45ab973653b442bf8a83d117 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 6 Feb 2016 01:22:27 +0100 Subject: [PATCH 21/30] update dependencies --- rtool.py | 2 +- setup.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rtool.py b/rtool.py index 78cf7bd9f..3e91ce249 100644 --- a/rtool.py +++ b/rtool.py @@ -210,7 +210,7 @@ def sdist(): @cli.command("bdist") @click.option("--use-existing-sdist/--no-use-existing-sdist", default=False) -@click.argument("pyinstaller_version", envvar="PYINSTALLER_VERSION", default="PyInstaller~=3.0.0") +@click.argument("pyinstaller_version", envvar="PYINSTALLER_VERSION", default="PyInstaller~=3.1.1") @click.pass_context def bdist(ctx, use_existing_sdist, pyinstaller_version): """ diff --git a/setup.py b/setup.py index e99c159cf..359b76463 100644 --- a/setup.py +++ b/setup.py @@ -5,10 +5,10 @@ setup( version='1.0', py_modules=['rtool'], install_requires=[ - 'click>=6.2', - 'twine>=1.6.4', - 'virtualenv>=13.1.2', - 'wheel>=0.26.0', + "click>=6.2, <7.0", + 'twine>=1.6.5, <1.7', + 'virtualenv>=14.0.5, <14.1', + 'wheel>=0.26.0, <0.27', ], entry_points={ 'console_scripts': [ From 5f079e56ec6e5d93d0acb744d2eead9cd1d42976 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 6 Feb 2016 01:23:55 +0100 Subject: [PATCH 22/30] update dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 359b76463..7af5273c7 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( "click>=6.2, <7.0", 'twine>=1.6.5, <1.7', 'virtualenv>=14.0.5, <14.1', - 'wheel>=0.26.0, <0.27', + 'wheel>=0.28.0, <0.29', ], entry_points={ 'console_scripts': [ From 9f48d5fb63df56b00169a0838e6a7b9611a67040 Mon Sep 17 00:00:00 2001 From: "requires.io" Date: Sat, 6 Feb 2016 18:25:01 +0100 Subject: [PATCH 23/30] [requires.io] dependency update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7af5273c7..195e0dfe2 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( "click>=6.2, <7.0", 'twine>=1.6.5, <1.7', 'virtualenv>=14.0.5, <14.1', - 'wheel>=0.28.0, <0.29', + 'wheel>=0.29,<0.30', ], entry_points={ 'console_scripts': [ From d3a9a6ba9520c5c80de3433697a8f5f30fa33c67 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 11 Feb 2016 18:51:47 +0100 Subject: [PATCH 24/30] add snapshot upload to rtool --- rtool.py | 182 +++++++++++++++++++++++++++++++++++++++++-------------- setup.py | 18 +++--- 2 files changed, 146 insertions(+), 54 deletions(-) diff --git a/rtool.py b/rtool.py index 3e91ce249..34dc2efdf 100644 --- a/rtool.py +++ b/rtool.py @@ -11,8 +11,9 @@ import runpy import zipfile import tarfile import platform -import sys import click +import pysftp +from six.moves import shlex_quote # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/ @@ -23,12 +24,12 @@ else: if platform.system() == "Windows": def Archive(name): - a = zipfile.ZipFile(name + ".zip", "w") + a = zipfile.ZipFile(name, "w") a.add = a.write return a else: def Archive(name): - return tarfile.open(name + ".tar.gz", "w:gz") + return tarfile.open(name, "w:gz") RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__))) DIST_DIR = join(RELEASE_DIR, "dist") @@ -46,17 +47,20 @@ ALL_PROJECTS = { "netlib": { "tools": [], "vfile": join(ROOT_DIR, "netlib/netlib/version.py"), - "dir": join(ROOT_DIR, "netlib") + "dir": join(ROOT_DIR, "netlib"), + "python_version": "py2.py3" # this is the format in wheel filenames }, "pathod": { "tools": ["pathod", "pathoc"], "vfile": join(ROOT_DIR, "pathod/libpathod/version.py"), - "dir": join(ROOT_DIR, "pathod") + "dir": join(ROOT_DIR, "pathod"), + "python_version": "py2" }, "mitmproxy": { "tools": ["mitmproxy", "mitmdump", "mitmweb"], "vfile": join(ROOT_DIR, "mitmproxy/libmproxy/version.py"), - "dir": join(ROOT_DIR, "mitmproxy") + "dir": join(ROOT_DIR, "mitmproxy"), + "python_version": "py2" } } if platform.system() == "Windows": @@ -69,6 +73,40 @@ def get_version(project): return runpy.run_path(projects[project]["vfile"])["VERSION"] +def get_snapshot_version(project): + last_tag, tag_dist, commit = subprocess.check_output( + ["git", "describe", "--tags", "--long"], + cwd=projects[project]["dir"] + ).strip().rsplit("-", 2) + tag_dist = int(tag_dist) + if tag_dist == 0: + return get_version(project) + else: + return "{version}dev{tag_dist:04}-{commit}".format( + version=get_version(project), # this should already be the next version + tag_dist=tag_dist, + commit=commit + ) + + +def archive_name(project): + platform_tag = { + "Darwin": "osx", + "Windows": "win32", + "Linux": "linux" + }.get(platform.system(), platform.system()) + if platform.system() == "Windows": + ext = "zip" + else: + ext = "tar.gz" + return "{project}-{version}-{platform}.{ext}".format( + project=project, + version=get_version(project), + platform=platform_tag, + ext=ext + ) + + def sdist_name(project): return "{project}-{version}.tar.gz".format( project=project, @@ -77,10 +115,10 @@ def sdist_name(project): def wheel_name(project): - return "{project}-{version}-py{py_version}-none-any.whl".format( + return "{project}-{version}-{py_version}-none-any.whl".format( project=project, version=get_version(project), - py_version=sys.version_info.major + py_version=projects[project]["python_version"] ) @@ -152,6 +190,14 @@ def set_version(version): f.write(new_content) +def _git(project, args): + print("%s> %s..." % (project, shlex_quote(args))) + subprocess.check_call( + args, + cwd=projects[project]["dir"] + ) + + @cli.command("git") @click.argument('args', nargs=-1, required=True) def git(args): @@ -160,11 +206,7 @@ def git(args): """ args = ["git"] + list(args) for project, conf in projects.items(): - print("%s> %s..." % (project, " ".join(args))) - subprocess.check_call( - args, - cwd=conf["dir"] - ) + _git(project, args) @cli.command("sdist") @@ -229,16 +271,7 @@ def bdist(ctx, use_existing_sdist, pyinstaller_version): for p, conf in projects.items(): if conf["tools"]: - archive_name = "{project}-{version}-{platform}".format( - project=p, - version=get_version(p), - platform={ - "Darwin": "osx", - "Windows": "win32", - "Linux": "linux" - }.get(platform.system(), platform.system()) - ) - with Archive(join(DIST_DIR, archive_name)) as archive: + with Archive(join(DIST_DIR, archive_name(p))) as archive: for tool in conf["tools"]: spec = join(conf["dir"], "release", "%s.spec" % tool) print("Building %s binary..." % tool) @@ -259,26 +292,29 @@ def bdist(ctx, use_existing_sdist, pyinstaller_version): executable = join(PYINSTALLER_DIST, tool) if platform.system() == "Windows": executable += ".exe" - print("Testinng %s..." % executable) + print("> %s --version" % executable) subprocess.check_call([executable, "--version"]) archive.add(executable, os.path.basename(executable)) - print("Packed {}.".format(archive_name)) + print("Packed {}.".format(archive_name(p))) -@cli.command("upload") +@cli.command("upload-release") @click.option('--username', prompt=True) @click.password_option(confirmation_prompt=False) @click.option('--repository', default="pypi") -def upload_release(username, password, repository): +@click.option("--sdist/--no-sdist", default=True) +@click.option("--wheel/--no-wheel", default=True) +def upload_release(username, password, repository, sdist, wheel): """ Upload source distributions to PyPI """ for project in projects.keys(): - files = ( - sdist_name(project), - wheel_name(project) - ) + files = [] + if sdist: + files.append(sdist_name(project)) + if wheel: + files.append(wheel_name(project)) for f in files: print("Uploading {} to {}...".format(f, repository)) subprocess.check_call([ @@ -291,13 +327,61 @@ def upload_release(username, password, repository): ]) +@cli.command("upload-snapshot") +@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True) +@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22) +@click.option("--username", envvar="SNAPSHOT_USER", prompt=True) +@click.option("--password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True) +@click.option("--sdist/--no-sdist", default=False) +@click.option("--wheel/--no-wheel", default=False) +@click.option("--bdist/--no-bdist", default=False) +def upload_snapshot(host, port, username, password, sdist, wheel, bdist): + """ + Upload snapshot to snapshot server + """ + with pysftp.Connection(host=host, port=port, username=username, password=password) as sftp: + for project, conf in projects.items(): + dir_name = "snapshots/v{}".format(get_version(project)) + sftp.makedirs(dir_name) + with sftp.cd(dir_name): + files = [] + if sdist: + files.append(sdist_name(project)) + if wheel: + files.append(wheel_name(project)) + if bdist and conf["tools"]: + files.append(archive_name(project)) + + for f in files: + local_path = join(DIST_DIR, f) + remote_filename = f.replace(get_version(project), get_snapshot_version(project)) + symlink_path = "../{}".format(f.replace(get_version(project), "latest")) + + print("Uploading {} as {}...".format(f, remote_filename)) + with click.progressbar(length=os.stat(local_path).st_size) as bar: + sftp.put( + local_path, + "." + remote_filename, + callback=lambda done, total: bar.update(done - bar.pos) + ) + # We hide the file during upload. + if sftp.exists(remote_filename): + sftp.remove(remote_filename) + sftp.rename("." + remote_filename, remote_filename) + + # add symlink + if sftp.lexists(symlink_path): + sftp.remove(symlink_path) + sftp.symlink("v{}/{}".format(get_version(project), remote_filename), symlink_path) + + @cli.command("wizard") -@click.option('--version', prompt=True) +@click.option('--next-version', prompt=True) @click.option('--username', prompt="PyPI Username") @click.password_option(confirmation_prompt=False, prompt="PyPI Password") @click.option('--repository', default="pypi") @click.pass_context -def wizard(ctx, version, username, password, repository): +def wizard(ctx, next_version, username, password, repository): """ Interactive Release Wizard """ @@ -306,29 +390,35 @@ def wizard(ctx, version, username, password, repository): if is_dirty: raise RuntimeError("%s repository is not clean." % project) - # bump version, update docs and contributors - ctx.invoke(set_version, version=version) + # update contributors file ctx.invoke(contributors) # Build test release ctx.invoke(bdist) - click.confirm("Please test the release now. Is it ok?", abort=True) - # version bump commit + tag - ctx.invoke( - git, args=["commit", "-a", "-m", "bump version"] - ) - ctx.invoke(git, args=["tag", "v" + version]) - ctx.invoke(git, args=["push"]) - ctx.invoke(git, args=["push", "--tags"]) + try: + click.confirm("Please test the release now. Is it ok?", abort=True) + except click.Abort: + # undo changes + ctx.invoke(git, ["checkout", "CONTRIBUTORS"]) + raise - # Re-invoke sdist with bumped version - ctx.invoke(sdist) - click.confirm("All good, can upload sdist to PyPI?", abort=True) + # Everything ok - let's ship it! + for p in projects.keys(): + _git(p, ["tag", "v" + get_version(p)]) + ctx.invoke(git, ["push", "--tags"]) ctx.invoke( upload_release, username=username, password=password, repository=repository ) + + # version bump commit + ctx.invoke(set_version, version=next_version) + ctx.invoke( + git, args=["commit", "-a", "-m", "bump version"] + ) + ctx.invoke(git, args=["push"]) + click.echo("All done!") diff --git a/setup.py b/setup.py index 7af5273c7..9876af0af 100644 --- a/setup.py +++ b/setup.py @@ -1,18 +1,20 @@ -from setuptools import setup, find_packages +from setuptools import setup setup( name='mitmproxy-rtool', - version='1.0', - py_modules=['rtool'], + version="1.0", + py_modules=["rtool"], install_requires=[ "click>=6.2, <7.0", - 'twine>=1.6.5, <1.7', - 'virtualenv>=14.0.5, <14.1', - 'wheel>=0.28.0, <0.29', + "twine>=1.6.5, <1.7", + "virtualenv>=14.0.5, <14.1", + "wheel>=0.29.0, <0.30", + "six>=1.10.0, <1.11", + "pysftp>=0.2.8, <0.3", ], entry_points={ - 'console_scripts': [ - 'rtool=rtool:cli', + "console_scripts": [ + "rtool=rtool:cli", ], }, ) From c75951d1cb2916a65235d0e5be5e5ea096ce33e9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 11 Feb 2016 19:34:29 +0100 Subject: [PATCH 25/30] use public key crypto for snapshot upload --- rtool.pem | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ rtool.py | 13 +++++++---- 2 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 rtool.pem diff --git a/rtool.pem b/rtool.pem new file mode 100644 index 000000000..097dff1a9 --- /dev/null +++ b/rtool.pem @@ -0,0 +1,68 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,7A4E07094E1AC0B74BB3D172A73A62DB + +0H4otaU3/McUviuQh/6L1tQM/gUjAc0nn1QESZ/n2R/VOlmLGwYPlMxHLXcs6a+y +byfQc/wcy1taPHSIVbEH1bYUfrFucLhlOq8jINPfsMqhPRGbH7DwTfAFNljZTGLU +V9zMXedIGujOdbJ29gClbYkaNbHB7wd+7icpRMJ/buEC5W9lZXpt5kspsG9Ueu6Z +T6rljxOQ5lyLun9E0y8H/f/RLViEyeEBSGK3UAHzz7+OJjKkKzVLfAFF2It4rfIh +PdFBhfyp3ihfs4Dy+eG2LzVSZC+D01uj0WkhGwIGzpB4XAgNjtbUeaz9o8649lLl +w4QuoX2cxbcLj4KVqSCml8aHA18oNtBdGx/s/+26bopmapcLwp01G51T/MMJTVpD +P+6GoFvHxE7YKasJV2fxQW70U3B8Ruu5ImZkKa9Dw7VI1XL0DY/WvQ46tYZqllyt +JeHxg21z0MeH6RiU7aT79tYB1KNcuuq9ujPFOSAED0C7TUEycvN4tyySPxqBAnBE +yHT0UU3AE1VpQJaYDlQWrZPC813WoPI8pjLw+v99f918TWOOQkRNfobLqjgOe/Km +cLT80vpaAjcwIZO9LTVYJ+gH/MDNLdgkIICAXL1MmgYBkhwuOw4+SxgfyK5gS4Pe +hkJjoc3oxkFUA7uZ/X4xY0t+vwPfXcdp3oDgYMWfjTH3NenFaa78+7u/udEVrzJ8 +jtLDOM7XmJk+8EWu4KOiCct2pAhbYHAXzGx8X+5gZRNgPhBBDpg5I0+YERgy08nx +QcBBBTxKEuOzgO+R1+9bucNzQYI61CRb+V4Sg2HjRsUP04J7TQDAeHZtpDbpIwey +OKqXaf4874Mi3CYktbO5ZP9VxlTHdb6cagxkzLpnFwIkEBAKI5MOMxmYyK+7vwrL +kn4dcNSReOLJaiLiMA9J3lswBPuVRRobnWCbLeHEq3a8j21JNZRBsZeKJCP1uPDR +AZbnk4DxXZVlSaISGhpj7dpPqIuGnMP4FnMORSBxssdB9F6q03Bh2Vl59U184Lvc +sbE5QvsSzJTQZc8790DTW90lwM3nfkJ+OePR4DwtTx1LdK8Z/mm6vll42Thwgsko +SKm6n47xgnVBm6Kmbl8G248RAsRbG/f1TQYKBlU8iBKeFocW2LDCIuftzgUqjgcz +5L/GlTbTzNPUT8VZ6WLtHBK6OBjebbE2zZ+W8mwlaVC+Z9Rush5Op4CO10Y1jK0P +cwAsG6G16h5nxtGVz/C028BG1sJS+2XPel12+CrN2rrb6qP41YyLtWJVzhj5jFBk +kAvf5g1w227Lt1CuRnICuPdP68MtzWuY5vQOE57eN/jVsuqmEHNwi8K0ZmtEwf5/ +Vq1QH/Nz/WP5vaIQGivtganAcTgdFD+HXmT0QS1qBot87w9LOU0JkMyBvaa5aXS9 +3QsVppnCn/x3OuE8SlujWvndJKb172u+0iR2mJ+OUuVMSjpfrSo5LQq03tmLOdD4 +mBg7WNuJOrjfyOq51S4zMrCqA2hoKL1lxE10RB+4RoHbpfpiblvlBVnY+cj6ic6t +8MuiXtKFcV4KwujKgB0aS2smLSkaA8tXa4k3vhgIm68FZRGpCMuT6J+dEWiRgY8j +FIWVlndlRustHN2qHyBipFm0Ei1iVK4qxcsLBhBI02ceXrcP5cFq8k3uuKJexKcf +EPXPEDy5AJKxjIKdhEiPZmMyvSQuN1Nwu97b+QG685IfMpB/18WBYwFrTJOEU7pP +a9uowuTaKXGugJIGhL6ziy5HN2zkKO2nZMp05/+nSXt4/j+C+pldG+rBNZF1SCPR +AmB5YySQHUDey3SZGZhHrTdGN85M6A2PnfWzdsxoOSE9pfag8nx1AUm5CJhyQTH+ +m2OgUNALXJczOot0v0hHxL0XFHxXmMobbqCbtveufwaiRlDdihQeEXK3obIvEs3P +bsb3OcKHPHFmmWluevX0wGsV9ZY+uSBvZaBRzmv5ij3n4CTpt+CVPn/IJq25E50Q +QZ4nkGGQyX5yQtHhIGjagfKCFRrnpuwgmVegWLCSVx3Rev+blpLygpz77D447CCx +uzAqTpAUiagv/U0av1B9U6fXwN5N8rN/qpbwQ/94nh+ELCuyBSEQHj6c9hIuy9pq +yPU5hKY7InYcNkI9U2lS2ZtEZsm/NEzeG3oIBdoPC0NlFUjDeWZkEXi1Vt9rlk7K +u3bOhqIVzxkeJIhde2D4CWMS9Kg8RFGhzB8nPOvCcU7vKukR5Ok/Q2bVtzPR3jRK +MZrARlCJTtCnkWhkYQlCuFpafuzrItITA4M8L1ZwjcVcvJNamq9b/URQfbOfsDY0 +8Igu/zuGGnb0DecPHX9DUVws/tk1HXpIC6RFiZFNfH7Pb63TyqF1dc+9oHbqRqfq +y05xekHY87Uzwyvn8Uch5j8dfoa3MvOWy/gEoK4ZMrsXyzywzLTHusaWnN172Oaz +G3IljaBijzzwMGkEXmPiO+4WM/mp8dv29GpNj0/Dr8L/hYGcZ87CzWGnSu93yToi +o1paUjM1zmK0CKHKda8FxaVUchIHEmi4DGxY1Ywj0NjeV5U3VQpLQawGm88aqdkq +2aF5annHPNfp5tncL3eySrZxy6Yy9hI1/CCxV9r/31JZdRofqwftgKyxrTN0TzXL +5fpvzZQly1B46yVzUUjPkez+t18Rr9i7Js/Dxop2+IHx1EDAlkeoz4OIDa6fUssp +30eVjsy4QTV65HUS2vIraoqvKHwiabqz+QU/k6EVmln5cM6fuqN4m9HBiGVVLTac +7WAzuW7D5mmIGXtl4JdoLXW9NuKAn7MIBixhlWY8/rdvBGRJ4zHQxM3RTmcAlaEA +1/BoQUNXundypyFO/dYH01N4N4TNJDXTpOslmJHc78c09w4PN6/MjQMF8SkTHCJ8 +eKPXxQOh7arK/ibyhT6cVE29BmBS2OAeIYxRpqVOxndnSv+dpH02L5NjgDi97YIB +bb2zMqyy+0ajQexov+PH1X2bxMLgY7GZ9zJ3GEyrq3MXYCLkR7IKm6R45xNMhFPO +VWx0CLZCFIiHbdUg1YPWhaqHK5fqIAlQSDgdurnWvvz3Y9gnbZxuzMQt4sAwiN/+ +2XM5ekMi50JEtiUyIggtLJl/0HAiGeAEld9Gbtl1uTQjngC1i6fLzvvBwUCQQ175 +-----END RSA PRIVATE KEY----- +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs0xeGx+bc+NCLGRJDr6y +qv825685l1Tb6lhLvYVYUFJfcxL1V79ML13CO09TPxGTbmKw7iI33ef9N2mUHHVe +24EjcSnYZ2Zx5zYIY4ejnDKCzM0md9XA3RpfTI9F3zGsJPt72veyHy6alCbF4NgX +FgAvp2UJmWwTLnJj3oIEwfGJeEcQRKjCFjSJlHGxbB6L0MSCgRtDxwlBfinzkriZ +0C8YyVv6u+DidHtobbZvwk/4uV3+KhoDfkpT2VE57a2KXr4c9Fekzr6szduFMkcI +KPqKouOOnaiNjOoKofbZ5I6cB6qoMkh+ebJvfPuEYBwhHmDDhpdjdk5v3LkeD5kt +0GtOaF5A3LbdZhgsC+1v1dHrL6Vndrb9KhO8ATHvlzJOonV73AoJFygp6upCvIt+ +tPUuy3VE0kjc2mJDDJr5V28pSBJ5r9uz1rXZWP+9aduR3M+RUGzHS7cUdsl+lHyQ +TFQAQIDhWXyokMGqVZBm9xJ2KcDiQAEr5+kP0BrhPmBtfHSnLA1tIXlFcOqakvyQ +q1cACMvwbkZpupTqMhCr2S4V9dKxnlO5EIWYJmme9moU/zu1KA0mcwqfnNeDOR2J +6c3/x3E+AqBoFzDQb6LQ2Iw+9bkKWKgDml4paLPu/ot8vA2ZB9/mujFUvp/1MLuD +5I1jQDXFbjAgAnaSyqyW368CAwEAAQ== +-----END PUBLIC KEY----- \ No newline at end of file diff --git a/rtool.py b/rtool.py index 34dc2efdf..2d6c6b590 100644 --- a/rtool.py +++ b/rtool.py @@ -330,16 +330,21 @@ def upload_release(username, password, repository, sdist, wheel): @cli.command("upload-snapshot") @click.option("--host", envvar="SNAPSHOT_HOST", prompt=True) @click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22) -@click.option("--username", envvar="SNAPSHOT_USER", prompt=True) -@click.option("--password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True) +@click.option("--user", envvar="SNAPSHOT_USER", prompt=True) +@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem")) +@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True) @click.option("--sdist/--no-sdist", default=False) @click.option("--wheel/--no-wheel", default=False) @click.option("--bdist/--no-bdist", default=False) -def upload_snapshot(host, port, username, password, sdist, wheel, bdist): +def upload_snapshot(host, port, user, private_key, private_key_password, sdist, wheel, bdist): """ Upload snapshot to snapshot server """ - with pysftp.Connection(host=host, port=port, username=username, password=password) as sftp: + with pysftp.Connection(host=host, + port=port, + username=user, + private_key=private_key, + private_key_pass=private_key_password) as sftp: for project, conf in projects.items(): dir_name = "snapshots/v{}".format(get_version(project)) sftp.makedirs(dir_name) From 16433ccb0ce3bfaf54294893ef32adec6359c2f6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 12 Feb 2016 21:57:48 +0100 Subject: [PATCH 26/30] automatically delete old snapshots --- rtool.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rtool.py b/rtool.py index 2d6c6b590..348fcd4a1 100644 --- a/rtool.py +++ b/rtool.py @@ -13,6 +13,7 @@ import tarfile import platform import click import pysftp +import fnmatch from six.moves import shlex_quote # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes @@ -362,6 +363,11 @@ def upload_snapshot(host, port, user, private_key, private_key_password, sdist, remote_filename = f.replace(get_version(project), get_snapshot_version(project)) symlink_path = "../{}".format(f.replace(get_version(project), "latest")) + old_version = f.replace(get_version(project), "*") + for f in sftp.listdir(): + if fnmatch.fnmatch(f, old_version): + sftp.remove(f) + print("Uploading {} as {}...".format(f, remote_filename)) with click.progressbar(length=os.stat(local_path).st_size) as bar: sftp.put( From b902f957dd5df7391a8f1029623ba4d8c805dee1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 12 Feb 2016 22:18:30 +0100 Subject: [PATCH 27/30] more verbose sftp output --- rtool.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rtool.py b/rtool.py index 348fcd4a1..b0f29b8bb 100644 --- a/rtool.py +++ b/rtool.py @@ -366,22 +366,23 @@ def upload_snapshot(host, port, user, private_key, private_key_password, sdist, old_version = f.replace(get_version(project), "*") for f in sftp.listdir(): if fnmatch.fnmatch(f, old_version): + print("Removing {}...".format(f)) sftp.remove(f) - print("Uploading {} as {}...".format(f, remote_filename)) + with click.progressbar(length=os.stat(local_path).st_size) as bar: + print("Uploading {} as {}...".format(f, remote_filename)) sftp.put( local_path, "." + remote_filename, callback=lambda done, total: bar.update(done - bar.pos) ) # We hide the file during upload. - if sftp.exists(remote_filename): - sftp.remove(remote_filename) sftp.rename("." + remote_filename, remote_filename) # add symlink if sftp.lexists(symlink_path): + print("Removing {}...".format(symlink_path)) sftp.remove(symlink_path) sftp.symlink("v{}/{}".format(get_version(project), remote_filename), symlink_path) From fb6cbbb7ace51f504a92fa4dbf6304a7ee6995d1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 14 Feb 2016 01:44:54 +0100 Subject: [PATCH 28/30] don't print while displaying a progress bar --- rtool.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rtool.py b/rtool.py index b0f29b8bb..de865b300 100644 --- a/rtool.py +++ b/rtool.py @@ -369,9 +369,8 @@ def upload_snapshot(host, port, user, private_key, private_key_password, sdist, print("Removing {}...".format(f)) sftp.remove(f) - + print("Uploading {} as {}...".format(f, remote_filename)) with click.progressbar(length=os.stat(local_path).st_size) as bar: - print("Uploading {} as {}...".format(f, remote_filename)) sftp.put( local_path, "." + remote_filename, From cf8588f1257e15c527168cebc6376651f9ddd7d7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 14 Feb 2016 17:28:25 +0100 Subject: [PATCH 29/30] fix minor bugs --- rtool.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rtool.py b/rtool.py index b0f29b8bb..858e9b056 100644 --- a/rtool.py +++ b/rtool.py @@ -192,9 +192,9 @@ def set_version(version): def _git(project, args): - print("%s> %s..." % (project, shlex_quote(args))) + print("%s> %s..." % (project, " ".join(shlex_quote(a) for a in args))) subprocess.check_call( - args, + ["git"] + list(args), cwd=projects[project]["dir"] ) @@ -205,9 +205,9 @@ def git(args): """ Run a git command on every project """ - args = ["git"] + list(args) for project, conf in projects.items(): _git(project, args) + print("") @cli.command("sdist") @@ -412,13 +412,13 @@ def wizard(ctx, next_version, username, password, repository): click.confirm("Please test the release now. Is it ok?", abort=True) except click.Abort: # undo changes - ctx.invoke(git, ["checkout", "CONTRIBUTORS"]) + ctx.invoke(git, args=["checkout", "CONTRIBUTORS"]) raise # Everything ok - let's ship it! for p in projects.keys(): _git(p, ["tag", "v" + get_version(p)]) - ctx.invoke(git, ["push", "--tags"]) + ctx.invoke(git, args=["push", "--tags"]) ctx.invoke( upload_release, username=username, password=password, repository=repository From 925c2062496f4d0860f9618e968b268bb282a275 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 14 Feb 2016 17:41:54 +0100 Subject: [PATCH 30/30] make sure that we don't have build race conditions --- rtool.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rtool.py b/rtool.py index dec0ee3f4..136b3066c 100644 --- a/rtool.py +++ b/rtool.py @@ -423,6 +423,8 @@ def wizard(ctx, next_version, username, password, repository): username=username, password=password, repository=repository ) + click.confirm("Now please wait until CI has built binaries. Finished?") + # version bump commit ctx.invoke(set_version, version=next_version) ctx.invoke(