Merge pull request #3131 from cortesi/postrelease

Post-release updates

CONTRIBUTORS

@@ -1,189 +0,0 @@
-  2407	Aldo Cortesi
-  1873	Maximilian Hils
-   556	Thomas Kriechbaumer
-   258	Shadab Zafar
-    97	Jason
-    83	Marcelo Glezer
-    68	Clemens
-    28	Jim Shaver
-    18	Henrik Nordstrom
-    16	Matthew Shao
-    14	Pedro Worcel
-    14	David Weinstein
-    13	Thomas Roth
-    11	Jake Drahos
-    11	Stephen Altamirano
-    11	arjun23496
-    11	Justus Wingert
-    10	András Veres-Szentkirályi
-    10	Zohar Lorberbaum
-    10	smill
-    10	Chris Czub
-    10	Sandor Nemes
-    10	Doug Freed
-     9	ikoz
-     9	Legend Tang
-     9	Rouli
-     9	Kyle Morton
-     8	Jason A. Novak
-     8	Chandler Abraham
-     7	Matthias Urlichs
-     7	Brad Peabody
-     7	dufferzafar
-     7	Alexis Hildebrandt
-     6	Felix Yan
-     5	Will Coster
-     5	Sam Cleveland
-     5	iroiro123
-     5	elitest
-     5	Tomaz Muraus
-     5	Choongwoo Han
-     4	Schamper
-     4	Youhei Sakurai
-     4	Bryan Bishop
-     4	root
-     4	Valtteri Virtanen
-     4	Clemens Brunner
-     4	Marc Liyanage
-     4	Wade 524
-     4	chhsiao90
-     4	yonder
-     4	Michael J. Bazzinotti
-     3	Ryan Welton
-     3	Ryan Laughlin
-     3	Kyle Manna
-     3	Eli Shvartsman
-     3	Vincent Haupert
-     3	Manish Kumar
-     3	Zack B
-     3	MatthewShao
-     3	redfast00
-     3	requires.io
-     3	Guillem Anguera
-     3	smill@cuckoo.sh
-     3	Chris Neasbitt
-     3	Benjamin Lee
-     2	Steven Van Acker
-     2	Slobodan Mišković
-     2	Jim Lloyd
-     2	isra17
-     2	israel
-     2	Sean Coates
-     2	Sachin Kelkar
-     2	jpkrause
-     2	Bennett Blodinger
-     2	lilydjwg
-     2	Michael Frister
-     2	Israel Nir
-     2	Cory Benfield
-     2	phackt
-     2	Anant
-     2	Jaime Soriano Pastor
-     2	Paul
-     2	Colin Bendell
-     2	依云
-     2	Heikki Hannikainen
-     2	Rob Wills
-     2	Niko Kommenda
-     2	Naveen Pai
-     2	strohu
-     2	alts
-     2	Yoginski
-     2	Mark E. Haase
-     2	Wade Catron
-     2	Terry Long
-     2	Krzysztof Bielicki
-     2	Nick Badger
-     1	Nicolas Esteves
-     1	Andrew Orr
-     1	Andrey Plotnikov
-     1	Andy Smith
-     1	Angelo Agatino Nicolosi
-     1	Anthony Zhang
-     1	BSalita
-     1	Ben Lerner
-     1	Bradley Baetz
-     1	Brady Law
-     1	Brett Randall
-     1	Chris Hamant
-     1	Christian Frichot
-     1	Dan Wilbraham
-     1	David Dworken
-     1	David Shaw
-     1	Doug Lethin
-     1	Drake Caraker
-     1	Edgar Boda-Majer
-     1	Eric Entzel
-     1	Felix Wolfsteller
-     1	FreeArtMan
-     1	Gabriel Kirkpatrick
-     1	Henrik Nordström
-     1	Israel Blancas
-     1	Ivaylo Popov
-     1	JC
-     1	Jakub Nawalaniec
-     1	Jakub Wilk
-     1	James Billingham
-     1	Jason Pepas
-     1	Jean Regisser
-     1	Jonathan Jones
-     1	Jorge Villacorta
-     1	Kit Randel
-     1	Kostya Esmukov
-     1	Linmiao Xu
-     1	Lucas Cimon
-     1	M. Utku Altinkaya
-     1	Mathieu Mitchell
-     1	Michael Bisbjerg
-     1	Mike C
-     1	Mike Fotinakis
-     1	Mikhail Korobov
-     1	Morton Fox
-     1	Nick HS
-     1	Nick Raptis
-     1	Aditya
-     1	Oleksandr Sheremet
-     1	Parth Ganatra
-     1	Pritam Baral
-     1	Quentin Pradet
-     1	Rich Somerfield
-     1	Rory McCann
-     1	Rune Halvorsen
-     1	Ryo Onodera
-     1	Sahil Chelaramani
-     1	Sahn Lam
-     1	Sanchit Sokhey
-     1	Seppo Yli-Olli
-     1	Sergey Chipiga
-     1	Stefan Wärting
-     1	Steve Phillips
-     1	Steven Noble
-     1	Suyash
-     1	Tai Dickerson
-     1	Tarashish Mishra
-     1	TearsDontFalls
-     1	Thiago Arrais
-     1	Tim Becker
-     1	Timothy Elliott
-     1	Tyler St. Onge
-     1	Ulrich Petri
-     1	Vyacheslav Bakhmutov
-     1	Wes Turner
-     1	Yuangxuan Wang
-     1	capt8bit
-     1	cle1000
-     1	davidpshaw
-     1	deployable
-     1	gecko655
-     1	jlhonora
-     1	joebowbeer
-     1	kronick
-     1	meeee
-     1	michaeljau
-     1	peralta
-     1	phil plante
-     1	sentient07
-     1	sethp-jive
-     1	starenka
-     1	vulnminer
-     1	vzvu3k6k

mitmproxy/version.py

@@ -3,7 +3,7 @@ import subprocess
 
 # The actual version string. For precompiled binaries, this will be changed to include the build
 # tag, e.g. "3.0.0.dev0042-0xcafeabc"
-VERSION = "4.0.0"
+VERSION = "5.0.0"
 PATHOD = "pathod " + VERSION
 MITMPROXY = "mitmproxy " + VERSION
 

release/README.md

@@ -1,26 +1,40 @@
 # Release Checklist
 
-Make sure run all these steps on the correct branch you want to create a new release for!
+Make sure to run all these steps on the correct branch you want to create a new
-- Verify `mitmproxy/version.py`
+release for! The command examples assume that you have a git remote called
+`upstream` that points to the `mitmproxy/mitmproxy` repo.
+
+- Verify that `mitmproxy/version.py` is correct
 - Update CHANGELOG
 - Verify that all CI tests pass
+- Create a major version branch - e.g. `v4.x`. Assuming you have a remote repo called `upstream` that points to the mitmproxy/mitmproxy repo::
+  - `git checkout -b v4.x upstream/master`
+  - `git push -u upstream v4.x`
 - Tag the release and push to Github
-  - For alphas, betas, and release candidates, use lightweight tags.
+  - For alphas, betas, and release candidates, use lightweight tags. This is
-    This is necessary so that the .devXXXX counter does not reset.
+    necessary so that the .devXXXX counter does not reset.
-  - For final releases, use annotated tags.
+  - For final releases, use annotated tags. This makes the .devXXXX counter reset.
-    This makes the .devXXXX counter reset.
+    - `git tag -a v4.0.0 -m v4.0.0`
+    - `git push upstream v4.0.0`
 - Wait for tag CI to complete
 
 ## GitHub Release
-- Create release notice on Github [here](https://github.com/mitmproxy/mitmproxy/releases/new)
+- Create release notice on Github
-- Attach all files from the new release folder on https://snapshots.mitmproxy.org
+  [here](https://github.com/mitmproxy/mitmproxy/releases/new) if not already
+  auto-created by the tag.
+- We DO NOT upload release artifacts to GitHub anymore. Simply add the
+  following snippet to the notice:
+  `You can find the latest release packages on our snapshot server: https://snapshots.mitmproxy.org/v<version number here>`
 
 ## PyPi
-- `tox -e rtool -- upload-release`
+- The created wheel is uploaded to PyPi automatically
+- Please check https://pypi.python.org/pypi/mitmproxy about the latest version
 
 ## Homebrew
-- `tox -e rtool -- homebrew-pr`
+- The Homebrew maintainers are typically very fast and detect our new relese
-- The Homebrew maintainers are typically very fast and detect our new relese within a day, but we can be a nice citizen and create the PR ourself.
+  within a day.
+- If you feel the need, you can run this from a macOS machine:
+  `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/v<version number here>`
 
 ## Docker
 - Update docker-releases repo
@@ -29,8 +43,8 @@ Make sure run all these steps on the correct branch you want to create a new rel
     * Creating a fresh venv, pip-installing the new wheel in there, and then export all packages:
     * `virtualenv -ppython3.6 venv && source venv/bin/activate && pip install mitmproxy && pip freeze`
   - Tag the commit with the correct version
-    * `2.0.0` for new major versions
+    * `v2.0.0` for new major versions
-    * `2.0.2` for new patch versions
+    * `v2.0.2` for new patch versions
 - Update `latest` tag [here](https://hub.docker.com/r/mitmproxy/mitmproxy/~/settings/automated-builds/)
 - Check that the build for this tag succeeds [https://hub.docker.com/r/mitmproxy/mitmproxy/builds/](here)
 - If build failed:
@@ -41,10 +55,17 @@ Make sure run all these steps on the correct branch you want to create a new rel
   - Check the build details page again
 
 ## Website
- - Update version here: https://github.com/mitmproxy/www/blob/master/src/config.toml
+ - Update version here:
- - `./build && ./upload-test`
+   https://github.com/mitmproxy/www/blob/master/src/config.toml
- - If everything looks alright: `./upload-prod`
+ - Run `./build && ./upload-test`
+ - If everything looks alright, run `./upload-prod`
+
+## Docs
+  - Make sure you've uploaded the previous version's docs to archive
+  - If everything looks alright:
+    - `./build-current`
+    - `./upload-stable`
 
 ## Prepare for next release
-
+ - Last but not least, bump the version on master in
- - Last but not least, bump the version on master in [https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) for major releases.
+   [https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) for major releases.

release/ci.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+import glob
+import re
 import contextlib
 import os
 import platform
@@ -10,7 +12,6 @@ import tarfile
 import zipfile
 from os.path import join, abspath, dirname, exists, basename
 
-import cryptography.fernet
 import click
 
 # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
@@ -73,17 +74,17 @@ TOOLS = [
 TAG = os.environ.get("TRAVIS_TAG", os.environ.get("APPVEYOR_REPO_TAG_NAME", None))
 BRANCH = os.environ.get("TRAVIS_BRANCH", os.environ.get("APPVEYOR_REPO_BRANCH", None))
 if TAG:
-    VERSION = TAG
+    VERSION = re.sub('^v', '', TAG)
     UPLOAD_DIR = VERSION
 elif BRANCH:
-    VERSION = BRANCH
+    VERSION = re.sub('^v', '', BRANCH)
     UPLOAD_DIR = "branches/%s" % VERSION
 else:
     print("Could not establish build name - exiting." % BRANCH)
     sys.exit(0)
 
-
 print("BUILD VERSION=%s" % VERSION)
+print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR)
 
 
 def archive_name(bdist: str) -> str:
@@ -99,23 +100,6 @@ def archive_name(bdist: str) -> str:
     )
 
 
-def wheel_name() -> str:
-    return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
-
-
-def installer_name() -> str:
-    ext = {
-        "Windows": "exe",
-        "Darwin": "dmg",
-        "Linux": "run"
-    }[platform.system()]
-    return "mitmproxy-{version}-{platform}-installer.{ext}".format(
-        version=VERSION,
-        platform=PLATFORM_TAG,
-        ext=ext,
-    )
-
-
 @contextlib.contextmanager
 def chdir(path: str):
     old_dir = os.getcwd()
@@ -134,7 +118,7 @@ def cli():
 
 @cli.command("info")
 def info():
-    print("Version: %s" % VERSION)
+    click.echo("Version: %s" % VERSION)
 
 
 @cli.command("build")
@@ -142,23 +126,41 @@ def build():
     """
     Build a binary distribution
     """
+    os.makedirs(DIST_DIR, exist_ok=True)
+    if "WHEEL" in os.environ:
+        build_wheel()
+    else:
+        click.echo("Not building wheels.")
+    build_pyinstaller()
+
+
+def build_wheel():
+    click.echo("Building wheel...")
+    subprocess.check_call([
+        "python",
+        "setup.py",
+        "-q",
+        "bdist_wheel",
+        "--dist-dir", DIST_DIR,
+    ])
+
+    whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
+    click.echo("Found wheel package: {}".format(whl))
+
+    subprocess.check_call([
+        "tox",
+        "-e", "wheeltest",
+        "--",
+        whl
+    ])
+
+
+def build_pyinstaller():
     if exists(PYINSTALLER_TEMP):
         shutil.rmtree(PYINSTALLER_TEMP)
     if exists(PYINSTALLER_DIST):
         shutil.rmtree(PYINSTALLER_DIST)
 
-    os.makedirs(DIST_DIR, exist_ok=True)
-
-    if "WHEEL" in os.environ:
-        print("Building wheel...")
-        subprocess.check_call(
-            [
-                "python",
-                "setup.py", "-q", "bdist_wheel",
-                "--dist-dir", "release/dist",
-            ]
-        )
-
     for bdist, tools in sorted(BDISTS.items()):
         with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
             for tool in tools:
@@ -168,7 +170,7 @@ def build():
                 # This is PyInstaller, so it messes up paths.
                 # We need to make sure that we are in the spec folder.
                 with chdir(PYINSTALLER_SPEC):
-                    print("Building %s binary..." % tool)
+                    click.echo("Building %s binary..." % tool)
                     excludes = []
                     if tool != "mitmweb":
                         excludes.append("mitmproxy.tools.web")
@@ -209,11 +211,11 @@ def build():
                     )
                     executable = executable.replace("_main", "")
 
-                print("> %s --version" % executable)
+                click.echo("> %s --version" % executable)
-                print(subprocess.check_output([executable, "--version"]).decode())
+                click.echo(subprocess.check_output([executable, "--version"]).decode())
 
                 archive.add(executable, basename(executable))
-        print("Packed {}.".format(archive_name(bdist)))
+        click.echo("Packed {}.".format(archive_name(bdist)))
 
 
 def is_pr():
@@ -229,34 +231,40 @@ def is_pr():
 @cli.command("upload")
 def upload():
     """
-        Upload snapshot to snapshot server
+        Upload build artifacts to snapshot server and
+        upload wheel package to PyPi
     """
     # This requires some explanation. The AWS access keys are only exposed to
     # privileged builds - that is, they are not available to PRs from forks.
     # However, they ARE exposed to PRs from a branch within the main repo. This
     # check catches that corner case, and prevents an inadvertent upload.
     if is_pr():
-        print("Refusing to upload a pull request")
+        click.echo("Refusing to upload a pull request")
         return
+
     if "AWS_ACCESS_KEY_ID" in os.environ:
-        subprocess.check_call(
+        subprocess.check_call([
-            [
+            "aws", "s3", "cp",
-                "aws", "s3", "cp",
+            "--acl", "public-read",
-                "--acl", "public-read",
+            DIST_DIR + "/",
-                DIST_DIR + "/",
+            "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
-                "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
+            "--recursive",
-                "--recursive",
+        ])
-            ]
-        )
 
-
+    upload_pypi = (
-@cli.command("decrypt")
+        TAG and
-@click.argument('infile', type=click.File('rb'))
+        "WHEEL" in os.environ and
-@click.argument('outfile', type=click.File('wb'))
+        "TWINE_USERNAME" in os.environ and
-@click.argument('key', envvar='RTOOL_KEY')
+        "TWINE_PASSWORD" in os.environ
-def decrypt(infile, outfile, key):
+    )
-    f = cryptography.fernet.Fernet(key.encode())
+    if upload_pypi:
-    outfile.write(f.decrypt(infile.read()))
+        filename = "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
+        click.echo("Uploading {} to PyPi...".format(filename))
+        subprocess.check_call([
+            "twine",
+            "upload",
+            join(DIST_DIR, filename)
+        ])
 
 
 if __name__ == "__main__":

release/rtool.py

@@ -1,114 +0,0 @@
-#!/usr/bin/env python3
-
-import contextlib
-import os
-import sys
-import platform
-import runpy
-import shlex
-import subprocess
-from os.path import join, abspath, dirname
-
-import cryptography.fernet
-import click
-
-
-ROOT_DIR = abspath(join(dirname(__file__), ".."))
-RELEASE_DIR = join(ROOT_DIR, "release")
-DIST_DIR = join(RELEASE_DIR, "dist")
-VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
-
-
-def git(args: str) -> str:
-    with chdir(ROOT_DIR):
-        return subprocess.check_output(["git"] + shlex.split(args)).decode()
-
-
-def get_version(dev: bool = False, build: bool = False) -> str:
-    x = runpy.run_path(VERSION_FILE)
-    return x["get_version"](dev, build, True)
-
-
-def wheel_name() -> str:
-    return "mitmproxy-{version}-py3-none-any.whl".format(
-        version=get_version(True),
-    )
-
-
-@contextlib.contextmanager
-def chdir(path: str):
-    old_dir = os.getcwd()
-    os.chdir(path)
-    yield
-    os.chdir(old_dir)
-
-
-@click.group(chain=True)
-def cli():
-    """
-    mitmproxy build tool
-    """
-    pass
-
-
-@cli.command("contributors")
-def contributors():
-    """
-    Update CONTRIBUTORS.md
-    """
-    with chdir(ROOT_DIR):
-        print("Updating CONTRIBUTORS...")
-        contributors_data = git("shortlog -n -s")
-        with open("CONTRIBUTORS", "wb") as f:
-            f.write(contributors_data.encode())
-
-
-@cli.command("upload-release")
-@click.option('--username', prompt=True)
-@click.password_option(confirmation_prompt=False)
-@click.option('--repository', default="pypi")
-def upload_release(username, password, repository):
-    """
-    Upload wheels to PyPI
-    """
-    filename = wheel_name()
-    print("Uploading {} to {}...".format(filename, repository))
-    subprocess.check_call([
-        "twine",
-        "upload",
-        "-u", username,
-        "-p", password,
-        "-r", repository,
-        join(DIST_DIR, filename)
-    ])
-
-
-@cli.command("homebrew-pr")
-def homebrew_pr():
-    """
-    Create a new Homebrew PR
-    """
-    if platform.system() != "Darwin":
-        print("You need to run this on macOS to create a new Homebrew PR. Sorry.")
-        sys.exit(1)
-
-    print("Creating a new PR with Homebrew...")
-    subprocess.check_call([
-        "brew",
-        "bump-formula-pr",
-        "--url", "https://github.com/mitmproxy/mitmproxy/archive/v{}".format(get_version()),
-        "mitmproxy",
-    ])
-
-
-@cli.command("encrypt")
-@click.argument('infile', type=click.File('rb'))
-@click.argument('outfile', type=click.File('wb'))
-@click.argument('key', envvar='RTOOL_KEY')
-def encrypt(infile, outfile, key):
-    f = cryptography.fernet.Fernet(key.encode())
-    outfile.write(f.encrypt(infile.read()))
-
-
-if __name__ == "__main__":
-    cli()

tox.ini

@@ -33,15 +33,27 @@ commands =
   python ./test/individual_coverage.py
 
 [testenv:cibuild]
-passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL
+passenv = TRAVIS_* AWS_* APPVEYOR_* TWINE_* RTOOL_KEY WHEEL
 deps =
   -rrequirements.txt
   pyinstaller==3.3.1
+  twine==1.11.0
   awscli
 commands =
   mitmdump --version
   python ./release/ci.py {posargs}
 
+[testenv:wheeltest]
+recreate = True
+deps =
+commands =
+  pip install {posargs}
+  mitmproxy --version
+  mitmdump --version
+  mitmweb --version
+  pathod --version
+  pathoc --version
+
 [testenv:docs]
 passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL
 deps =