clean up release tool, build linux binaries

This commit is contained in:
Maximilian Hils 2016-07-29 18:37:37 -07:00
parent e800f321af
commit 3b71c19af3
4 changed files with 165 additions and 185 deletions

View File

@ -31,12 +31,12 @@ deploy_script:
ps: |
if(
($Env:TOXENV -match "py35") -and
(($Env:APPVEYOR_REPO_BRANCH -match "master") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
(($Env:APPVEYOR_REPO_BRANCH -match "builds") -or ($Env:APPVEYOR_REPO_TAG -match "true"))
) {
pip install -U virtualenv
.\dev.ps1
cmd /c "python .\release\rtool.py bdist 2>&1"
python .\release\rtool.py upload-snapshot --bdist
cmd /c "python -u .\release\rtool.py bdist 2>&1"
python -u .\release\rtool.py upload-snapshot --bdist --wheel
}
cache:

View File

@ -23,9 +23,9 @@ matrix:
- os: osx
osx_image: xcode7.3
language: generic
env: TOXENV=py35
env: TOXENV=py35 BDIST=1
- python: 3.5
env: TOXENV=py35
env: TOXENV=py35 BDIST=1
- python: 3.5
env: TOXENV=py35 NO_ALPN=1
- python: 2.7
@ -57,14 +57,14 @@ script: set -o pipefail; python -m tox -- --cov netlib --cov mitmproxy --cov pat
after_success:
- |
if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "builds" || -n $TRAVIS_TAG) ]]
then
git fetch --unshallow
./dev.sh 3.5
source venv3.5/bin/activate
pip install -e ./release
python ./release/rtool.py bdist
python ./release/rtool.py upload-snapshot --bdist --wheel
python -u ./release/rtool.py bdist
python -u ./release/rtool.py upload-snapshot --bdist
fi
notifications:

View File

@ -1,10 +1,13 @@
1813 Aldo Cortesi
1228 Maximilian Hils
282 Thomas Kriechbaumer
2118 Aldo Cortesi
1666 Maximilian Hils
450 Thomas Kriechbaumer
210 Shadab Zafar
94 Jason
83 Marcelo Glezer
36 Clemens
28 Jim Shaver
18 Henrik Nordstrom
17 Shadab Zafar
16 Matthew Shao
14 David Weinstein
14 Pedro Worcel
13 Thomas Roth
@ -14,39 +17,47 @@
10 András Veres-Szentkirályi
10 Chris Czub
10 Sandor Nemes
10 Zohar Lorberbaum
9 Kyle Morton
9 Legend Tang
9 Matthew Shao
9 Rouli
9 ikoz
8 Chandler Abraham
8 Jason A. Novak
7 Alexis Hildebrandt
7 Brad Peabody
7 Matthias Urlichs
7 dufferzafar
6 Felix Yan
5 Choongwoo Han
5 Sam Cleveland
5 Tomaz Muraus
5 Will Coster
5 elitest
5 iroiro123
4 Bryan Bishop
4 Clemens Brunner
4 Marc Liyanage
4 Michael J. Bazzinotti
4 Valtteri Virtanen
4 Wade 524
4 Youhei Sakurai
4 root
4 yonder
3 Benjamin Lee
3 Chris Neasbitt
3 Eli Shvartsman
3 Felix Yan
3 Guillem Anguera
3 Kyle Manna
3 MatthewShao
3 Ryan Welton
3 Zack B
3 redfast00
3 requires.io
2 Anant
2 Bennett Blodinger
2 Colin Bendell
2 Cory Benfield
2 Heikki Hannikainen
2 Israel Nir
2 Jaime Soriano Pastor
@ -59,34 +70,50 @@
2 Paul
2 Rob Wills
2 Sean Coates
2 Steven Van Acker
2 Terry Long
2 Wade Catron
2 alts
2 isra17
2 israel
2 requires.io
2 jpkrause
2 lilydjwg
2 strohu
2 依云
1 Aditya
1 Andrey Plotnikov
1 Andy Smith
1 Anthony Zhang
1 BSalita
1 Ben Lerner
1 Bradley Baetz
1 Brett Randall
1 Chris Hamant
1 Christian Frichot
1 Dan Wilbraham
1 David Dworken
1 David Shaw
1 Doug Freed
1 Doug Lethin
1 Drake Caraker
1 Eric Entzel
1 Felix Wolfsteller
1 FreeArtMan
1 Gabriel Kirkpatrick
1 Henrik Nordström
1 Israel Blancas
1 Ivaylo Popov
1 JC
1 Jakub Nawalaniec
1 Jakub Wilk
1 James Billingham
1 Jason Pepas
1 Jean Regisser
1 Jonathan Jones
1 Jorge Villacorta
1 Kit Randel
1 Kostya Esmukov
1 Linmiao Xu
1 Lucas Cimon
1 M. Utku Altinkaya
1 Mathieu Mitchell
@ -98,27 +125,33 @@
1 Nick Raptis
1 Nicolas Esteves
1 Oleksandr Sheremet
1 Parth Ganatra
1 Pritam Baral
1 Rich Somerfield
1 Rory McCann
1 Rune Halvorsen
1 Ryo Onodera
1 Sachin Kelkar
1 Sahn Lam
1 Seppo Yli-Olli
1 Sergey Chipiga
1 Stefan Wärting
1 Steve Phillips
1 Steven Van Acker
1 Steven Noble
1 Suyash
1 Tai Dickerson
1 Tarashish Mishra
1 TearsDontFalls
1 Thiago Arrais
1 Tim Becker
1 Timothy Elliott
1 Ulrich Petri
1 Vyacheslav Bakhmutov
1 Will Coster
1 Wes Turner
1 Yoginski
1 Yuangxuan Wang
1 capt8bit
1 cle1000
1 davidpshaw
1 deployable
1 gecko655
@ -133,4 +166,3 @@
1 sethp-jive
1 starenka
1 vzvu3k6k
1 依云

View File

@ -1,29 +1,30 @@
#!/usr/bin/env python
#!/usr/bin/env python3
from __future__ import absolute_import, print_function, division
from os.path import join
import contextlib
import fnmatch
import os
import platform
import runpy
import shlex
import shutil
import subprocess
import re
import shlex
import runpy
import zipfile
import sys
import tarfile
import platform
import zipfile
from os.path import join, normpath, dirname, exists, basename
import click
import pysftp
import fnmatch
# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
import sys
if platform.system() == "Windows":
VENV_BIN = "Scripts"
else:
VENV_BIN = "bin"
# ZipFile and tarfile have slightly different APIs. Fix that.
if platform.system() == "Windows":
def Archive(name):
a = zipfile.ZipFile(name, "w")
@ -33,13 +34,13 @@ else:
def Archive(name):
return tarfile.open(name, "w:gz")
RELEASE_DIR = join(os.path.dirname(os.path.realpath(__file__)))
DIST_DIR = join(RELEASE_DIR, "dist")
ROOT_DIR = os.path.normpath(join(RELEASE_DIR, ".."))
RELEASE_SPEC_DIR = join(RELEASE_DIR, "specs")
VERSION_FILE = join(ROOT_DIR, "netlib/version.py")
ROOT_DIR = normpath(join(dirname(__file__), ".."))
RELEASE_DIR = join(ROOT_DIR, "release")
BUILD_DIR = join(RELEASE_DIR, "build")
DIST_DIR = join(RELEASE_DIR, "dist")
PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
PYINSTALLER_DIST = join(BUILD_DIR, "binaries")
@ -47,27 +48,35 @@ VENV_DIR = join(BUILD_DIR, "venv")
VENV_PIP = join(VENV_DIR, VENV_BIN, "pip")
VENV_PYINSTALLER = join(VENV_DIR, VENV_BIN, "pyinstaller")
project = {
"name": "mitmproxy",
"tools": ["pathod", "pathoc", "mitmproxy", "mitmdump", "mitmweb"],
"bdists": {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"]
},
"dir": ROOT_DIR,
"python_version": "py2.py3",
# Project Configuration
VERSION_FILE = join(ROOT_DIR, "netlib", "version.py")
PROJECT_NAME = "mitmproxy"
PYTHON_VERSION = "py2.py3"
BDISTS = {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"]
}
if platform.system() == "Windows":
project["tools"].remove("mitmproxy")
project["bdists"]["mitmproxy"].remove("mitmproxy")
BDISTS["mitmproxy"].remove("mitmproxy")
TOOLS = [
tool
for tools in BDISTS.values()
for tool in tools
]
def get_version():
def get_version() -> str:
return runpy.run_path(VERSION_FILE)["VERSION"]
def get_snapshot_version():
last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit(b"-", 2)
def git(args: str) -> str:
with chdir(ROOT_DIR):
return subprocess.check_output(["git"] + shlex.split(args)).decode()
def get_snapshot_version() -> str:
last_tag, tag_dist, commit = git("describe --tags --long").strip().rsplit("-", 2)
tag_dist = int(tag_dist)
if tag_dist == 0:
return get_version()
@ -76,11 +85,11 @@ def get_snapshot_version():
return "{version}dev{tag_dist:04}-0x{commit}".format(
version=get_version(), # this should already be the next version
tag_dist=tag_dist,
commit=commit.decode()
commit=commit
)
def archive_name(project):
def archive_name(bdist: str) -> str:
platform_tag = {
"Darwin": "osx",
"Windows": "win32",
@ -91,18 +100,18 @@ def archive_name(project):
else:
ext = "tar.gz"
return "{project}-{version}-{platform}.{ext}".format(
project=project,
project=bdist,
version=get_version(),
platform=platform_tag,
ext=ext
)
def wheel_name():
def wheel_name() -> str:
return "{project}-{version}-{py_version}-none-any.whl".format(
project=project["name"],
project=PROJECT_NAME,
version=get_version(),
py_version=project["python_version"]
py_version=PYTHON_VERSION
)
@ -119,18 +128,13 @@ def empty_pythonpath():
@contextlib.contextmanager
def chdir(path):
def chdir(path: str):
old_dir = os.getcwd()
os.chdir(path)
yield
os.chdir(old_dir)
def git(args):
with chdir(ROOT_DIR):
return subprocess.check_output(["git"] + shlex.split(args))
@click.group(chain=True)
def cli():
"""
@ -147,95 +151,79 @@ def contributors():
with chdir(ROOT_DIR):
print("Updating CONTRIBUTORS...")
contributors_data = git("shortlog -n -s")
with open("CONTRIBUTORS", "w") as f:
f.write(contributors_data)
with open("CONTRIBUTORS", "wb") as f:
f.write(contributors_data.encode())
@cli.command("set-version")
@click.argument('version')
def set_version(version):
@cli.command("wheel")
def make_wheel():
"""
Update version information
"""
print("Update versions...")
version = ", ".join(version.split("."))
print("Update %s..." % VERSION_FILE)
with open(VERSION_FILE, "rb") as f:
content = f.read()
new_content = re.sub(
r"IVERSION\s*=\s*\([\d,\s]+\)", "IVERSION = (%s)" % version,
content
)
with open(VERSION_FILE, "wb") as f:
f.write(new_content)
@cli.command("wheels")
def wheels():
"""
Build wheels
Build wheel
"""
with empty_pythonpath():
print("Building release...")
if os.path.exists(DIST_DIR):
if exists(DIST_DIR):
shutil.rmtree(DIST_DIR)
print("Creating wheel for %s ..." % project["name"])
print("Creating wheel...")
subprocess.check_call(
[
"python", "./setup.py", "-q",
"bdist_wheel", "--dist-dir", DIST_DIR, "--universal"
],
cwd=project["dir"]
cwd=ROOT_DIR
)
print("Creating virtualenv for test install...")
if os.path.exists(VENV_DIR):
if exists(VENV_DIR):
shutil.rmtree(VENV_DIR)
subprocess.check_call(["virtualenv", "-q", VENV_DIR])
with chdir(DIST_DIR):
print("Installing %s..." % project["name"])
print("Install wheel into virtualenv...")
# lxml...
if platform.system() == "Windows" and sys.version_info[0] == 3:
subprocess.check_call([VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"])
subprocess.check_call(
[VENV_PIP, "install", "-q", "https://snapshots.mitmproxy.org/misc/lxml-3.6.0-cp35-cp35m-win32.whl"]
)
subprocess.check_call([VENV_PIP, "install", "-q", wheel_name()])
print("Running binaries...")
for tool in project["tools"]:
print("Running tools...")
for tool in TOOLS:
tool = join(VENV_DIR, VENV_BIN, tool)
print("> %s --version" % tool)
print(subprocess.check_output([tool, "--version"]))
print(subprocess.check_output([tool, "--version"]).decode())
print("Virtualenv available for further testing:")
print("source %s" % os.path.normpath(join(VENV_DIR, VENV_BIN, "activate")))
print("source %s" % normpath(join(VENV_DIR, VENV_BIN, "activate")))
@cli.command("bdist")
@click.option("--use-existing-wheels/--no-use-existing-wheels", default=False)
@click.option("--use-existing-wheel/--no-use-existing-wheel", default=False)
@click.argument("pyinstaller_version", envvar="PYINSTALLER_VERSION", default="PyInstaller~=3.1.1")
@click.argument("setuptools_version", envvar="SETUPTOOLS_VERSION", default="setuptools>=25.1.0,!=25.1.1")
@click.pass_context
def bdist(ctx, use_existing_wheels, pyinstaller_version):
def make_bdist(ctx, use_existing_wheel, pyinstaller_version, setuptools_version):
"""
Build a binary distribution
"""
if os.path.exists(PYINSTALLER_TEMP):
if exists(PYINSTALLER_TEMP):
shutil.rmtree(PYINSTALLER_TEMP)
if os.path.exists(PYINSTALLER_DIST):
if exists(PYINSTALLER_DIST):
shutil.rmtree(PYINSTALLER_DIST)
if not use_existing_wheels:
ctx.invoke(wheels)
if not use_existing_wheel:
ctx.invoke(make_wheel)
print("Installing PyInstaller...")
subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version])
print("Installing PyInstaller and setuptools...")
subprocess.check_call([VENV_PIP, "install", "-q", pyinstaller_version, setuptools_version])
print(subprocess.check_output([VENV_PIP, "freeze"]).decode())
for bdist_project, tools in project["bdists"].items():
with Archive(join(DIST_DIR, archive_name(bdist_project))) as archive:
for bdist, tools in BDISTS.items():
with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
for tool in tools:
# This is PyInstaller, so it messes up paths.
# We need to make sure that we are in the spec folder.
with chdir(RELEASE_SPEC_DIR):
with chdir(PYINSTALLER_SPEC):
print("Building %s binary..." % tool)
subprocess.check_call(
[
@ -255,10 +243,10 @@ def bdist(ctx, use_existing_wheels, pyinstaller_version):
if platform.system() == "Windows":
executable += ".exe"
print("> %s --version" % executable)
subprocess.check_call([executable, "--version"])
print(subprocess.check_output([executable, "--version"]).decode())
archive.add(executable, os.path.basename(executable))
print("Packed {}.".format(archive_name(bdist_project)))
archive.add(executable, basename(executable))
print("Packed {}.".format(archive_name(bdist)))
@cli.command("upload-release")
@ -298,90 +286,50 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel,
username=user,
private_key=private_key,
private_key_pass=private_key_password) as sftp:
dir_name = "snapshots/v{}".format(get_version())
sftp.makedirs(dir_name)
with sftp.cd(dir_name):
files = []
if wheel:
files.append(wheel_name())
for bdist in project["bdists"].keys():
dir_name = "snapshots/v{}".format(get_version())
sftp.makedirs(dir_name)
with sftp.cd(dir_name):
files = []
if wheel:
files.append(wheel_name())
if bdist:
for bdist in BDISTS.keys():
files.append(archive_name(bdist))
for f in files:
local_path = join(DIST_DIR, f)
remote_filename = f.replace(get_version(), get_snapshot_version())
symlink_path = "../{}".format(f.replace(get_version(), "latest"))
for f in files:
local_path = join(DIST_DIR, f)
remote_filename = f.replace(get_version(), get_snapshot_version())
symlink_path = "../{}".format(f.replace(get_version(), "latest"))
# Delete old versions
old_version = f.replace(get_version(), "*")
for f_old in sftp.listdir():
if fnmatch.fnmatch(f_old, old_version):
print("Removing {}...".format(f_old))
sftp.remove(f_old)
# Upload new version
print("Uploading {} as {}...".format(f, remote_filename))
with click.progressbar(length=os.stat(local_path).st_size) as bar:
# We hide the file during upload
sftp.put(
local_path,
"." + remote_filename,
callback=lambda done, total: bar.update(done - bar.pos)
)
# Upload new version
print("Uploading {} as {}...".format(f, remote_filename))
with click.progressbar(length=os.stat(local_path).st_size) as bar:
sftp.put(
local_path,
"." + remote_filename,
callback=lambda done, total: bar.update(done - bar.pos)
)
# We hide the file during upload.
sftp.rename("." + remote_filename, remote_filename)
# Delete old versions
old_version = f.replace(get_version(), "*")
for f_old in sftp.listdir():
if fnmatch.fnmatch(f_old, old_version):
print("Removing {}...".format(f_old))
sftp.remove(f_old)
# update symlink for the latest release
if sftp.lexists(symlink_path):
print("Removing {}...".format(symlink_path))
sftp.remove(symlink_path)
# Show new version
sftp.rename("." + remote_filename, remote_filename)
# update symlink for the latest release
if sftp.lexists(symlink_path):
print("Removing {}...".format(symlink_path))
sftp.remove(symlink_path)
if f != wheel_name():
# "latest" isn't a proper wheel version, so this could not be installed.
# https://github.com/mitmproxy/mitmproxy/issues/1065
sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path)
@cli.command("wizard")
@click.option('--next-version', prompt=True)
@click.option('--username', prompt="PyPI Username")
@click.password_option(confirmation_prompt=False, prompt="PyPI Password")
@click.option('--repository', default="pypi")
@click.pass_context
def wizard(ctx, next_version, username, password, repository):
"""
Interactive Release Wizard
"""
is_dirty = git("status --porcelain")
if is_dirty:
raise RuntimeError("Repository is not clean.")
# update contributors file
ctx.invoke(contributors)
# Build test release
ctx.invoke(bdist)
try:
click.confirm("Please test the release now. Is it ok?", abort=True)
except click.Abort:
# undo changes
git("checkout CONTRIBUTORS")
raise
# Everything ok - let's ship it!
git("tag v{}".format(get_version()))
git("push --tags")
ctx.invoke(
upload_release,
username=username, password=password, repository=repository
)
click.confirm("Now please wait until CI has built binaries. Finished?")
# version bump commit
ctx.invoke(set_version, version=next_version)
git("commit -a -m \"bump version\"")
git("push")
click.echo("All done!")
if __name__ == "__main__":
cli()