release: revamped release upload process

.travis.yml

@@ -18,15 +18,15 @@ matrix:
   include:
     # - python: 3.6
     #   env: TOXENV=lint
-    # - os: osx
-    #   osx_image: xcode7.3
-    #   language: generic
-    #   env: TOXENV=py35 BDIST=1
+    - os: osx
+      osx_image: xcode7.3
+      language: generic
+      env: TOXENV=py36 BDIST=1
     # - python: 3.5
-    #   env: TOXENV=py35 BDIST=1
+    #   env: TOXENV=py35
     #   dist: precise
     # - python: 3.6
-    #   env: TOXENV=py36
+    #   env: TOXENV=py36 BDIST=1
     # - python: 3.6
     #   env: TOXENV=individual_coverage
     # - language: node_js
@@ -42,17 +42,17 @@ matrix:
     #       yarn: true
     #       directories:
     #           - web/node_modules
-    - language: python
-      env:
-        - NAME=docs
-      install:
-        - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb
-        - sudo dpkg -i hugo*.deb
-        - pip install awscli
-      script:
-        - cd docs && ./ci
-      after_success:
-        - echo done
+    # - language: python
+    #   env:
+    #     - NAME=docs
+    #   install:
+    #     - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb
+    #     - sudo dpkg -i hugo*.deb
+    #     - pip install awscli
+    #   script:
+    #     - cd docs && ./ci
+    #   after_success:
+    #     - echo done
 
 install:
   - |
@@ -61,12 +61,11 @@ install:
       brew update || brew update
       brew outdated pyenv || brew upgrade pyenv
       eval "$(pyenv init -)"
-      env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.5.3
-      pyenv global 3.5.3
-      pyenv shell 3.5.3
-      pip install -U pip setuptools wheel virtualenv
+      env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.4
+      pyenv global 3.6.4
+      pyenv shell 3.6.4
     fi
-  - pip install tox
+  - pip install tox virtualenv setuptools
 
 script:
   - tox -- --verbose --cov-report=term
@@ -74,16 +73,14 @@ script:
     if [[ $BDIST == "1" ]]
     then
         git fetch --unshallow --tags
-        tox -e rtool -- bdist
+        tox -e cibuild -- build
     fi
 
 after_success:
-  # we build binaries on every run, but we only upload them for master snapshots or tags.
   - |
-    if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]]
+    if [[ $BDIST == "1" ]]
     then
-        tox -e rtool -- decrypt release/known_hosts.enc release/known_hosts
-        tox -e rtool -- upload-snapshot --bdist
+        tox -e cibuild -- upload
     fi
 
 notifications:

docs/ci

@@ -6,7 +6,7 @@
 
 # Only upload if we have defined credentials - we only have these defined for
 # trusted commits (i.e. not PRs).
-if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "updocs" ]]; then
+if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then
     aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master
     # aws cloudfront create-invalidation --distribution-id E3UCZ4MLN4TO7U \
     #     --paths "/master"

release/ci.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+
+import contextlib
+import os
+import platform
+import re
+import sys
+import shutil
+import subprocess
+import tarfile
+import zipfile
+from os.path import join, abspath, dirname, exists, basename
+
+import click
+
+# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
+# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
+if platform.system() == "Windows":
+    VENV_BIN = "Scripts"
+    PYINSTALLER_ARGS = [
+        # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
+        "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
+    ]
+else:
+    VENV_BIN = "bin"
+    PYINSTALLER_ARGS = []
+
+# ZipFile and tarfile have slightly different APIs. Fix that.
+if platform.system() == "Windows":
+    def Archive(name):
+        a = zipfile.ZipFile(name, "w")
+        a.add = a.write
+        return a
+else:
+    def Archive(name):
+        return tarfile.open(name, "w:gz")
+
+PLATFORM_TAG = {
+    "Darwin": "osx",
+    "Windows": "windows",
+    "Linux": "linux",
+}.get(platform.system(), platform.system())
+
+ROOT_DIR = abspath(join(dirname(__file__), ".."))
+RELEASE_DIR = join(ROOT_DIR, "release")
+
+BUILD_DIR = join(RELEASE_DIR, "build")
+DIST_DIR = join(RELEASE_DIR, "dist")
+
+PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
+# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
+PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
+PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
+PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
+
+VENV_DIR = join(BUILD_DIR, "venv")
+
+# Project Configuration
+SNAPSHOT_BRANCHES = ["master", "updocs"]
+VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
+BDISTS = {
+    "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
+    "pathod": ["pathoc", "pathod"]
+}
+if platform.system() == "Windows":
+    BDISTS["mitmproxy"].remove("mitmproxy")
+
+TOOLS = [
+    tool
+    for tools in sorted(BDISTS.values())
+    for tool in tools
+]
+
+if os.environ.get("TRAVIS_TAG", None):
+    VERSION = os.environ["TRAVIS_TAG"]
+elif os.environ.get("TRAVIS_BRANCH", None) in SNAPSHOT_BRANCHES:
+    VERSION = os.environ["TRAVIS_BRANCH"]
+else:
+    print("Branch %s is not buildabranch - exiting." % os.environ.get("TRAVIS_BRANCH", None))
+    sys.exit(0)
+
+print("BUILD VERSION=%s" % VERSION)
+
+
+def set_version(dev: bool) -> None:
+    """
+    Update version information in mitmproxy's version.py to either include hardcoded information or not.
+    """
+    version = get_version(dev, dev)
+    with open(VERSION_FILE, "r") as f:
+        content = f.read()
+    content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M)
+    with open(VERSION_FILE, "w") as f:
+        f.write(content)
+
+
+def archive_name(bdist: str) -> str:
+    if platform.system() == "Windows":
+        ext = "zip"
+    else:
+        ext = "tar.gz"
+    return "{project}-{version}-{platform}.{ext}".format(
+        project=bdist,
+        version=VERSION,
+        platform=PLATFORM_TAG,
+        ext=ext
+    )
+
+
+def wheel_name() -> str:
+    return "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
+
+
+def installer_name() -> str:
+    ext = {
+        "Windows": "exe",
+        "Darwin": "dmg",
+        "Linux": "run"
+    }[platform.system()]
+    return "mitmproxy-{version}-{platform}-installer.{ext}".format(
+        version=VERSION,
+        platform=PLATFORM_TAG,
+        ext=ext,
+    )
+
+
+@contextlib.contextmanager
+def chdir(path: str):
+    old_dir = os.getcwd()
+    os.chdir(path)
+    yield
+    os.chdir(old_dir)
+
+
+@click.group(chain=True)
+def cli():
+    """
+    mitmproxy build tool
+    """
+    pass
+
+@cli.command("info")
+def info():
+    print("Version: %s" % VERSION)
+
+
+@cli.command("wheel")
+def make_wheel():
+    """
+    Build a Python wheel
+    """
+    set_version(True)
+    try:
+        subprocess.check_call([
+            "tox", "-e", "wheel",
+        ], env={
+            **os.environ,
+            "VERSION": VERSION,
+        })
+    finally:
+        set_version(False)
+
+
+@cli.command("build")
+def build():
+    """
+    Build a binary distribution
+    """
+    if exists(PYINSTALLER_TEMP):
+        shutil.rmtree(PYINSTALLER_TEMP)
+    if exists(PYINSTALLER_DIST):
+        shutil.rmtree(PYINSTALLER_DIST)
+
+    os.makedirs(DIST_DIR, exist_ok=True)
+
+    for bdist, tools in sorted(BDISTS.items()):
+        with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
+            for tool in tools:
+                # We can't have a folder and a file with the same name.
+                if tool == "mitmproxy":
+                    tool = "mitmproxy_main"
+                # This is PyInstaller, so it messes up paths.
+                # We need to make sure that we are in the spec folder.
+                with chdir(PYINSTALLER_SPEC):
+                    print("Building %s binary..." % tool)
+                    excludes = []
+                    if tool != "mitmweb":
+                        excludes.append("mitmproxy.tools.web")
+                    if tool != "mitmproxy_main":
+                        excludes.append("mitmproxy.tools.console")
+
+                    subprocess.check_call(
+                        [
+                            "pyinstaller",
+                            "--clean",
+                            "--workpath", PYINSTALLER_TEMP,
+                            "--distpath", PYINSTALLER_DIST,
+                            "--additional-hooks-dir", PYINSTALLER_HOOKS,
+                            "--onefile",
+                            "--console",
+                            "--icon", "icon.ico",
+                            # This is PyInstaller, so setting a
+                            # different log level obviously breaks it :-)
+                            # "--log-level", "WARN",
+                        ]
+                        + [x for e in excludes for x in ["--exclude-module", e]]
+                        + PYINSTALLER_ARGS
+                        + [tool]
+                    )
+                    # Delete the spec file - we're good without.
+                    os.remove("{}.spec".format(tool))
+
+                # Test if it works at all O:-)
+                executable = join(PYINSTALLER_DIST, tool)
+                if platform.system() == "Windows":
+                    executable += ".exe"
+
+                # Remove _main suffix from mitmproxy executable
+                if "_main" in executable:
+                    shutil.move(
+                        executable,
+                        executable.replace("_main", "")
+                    )
+                    executable = executable.replace("_main", "")
+
+                print("> %s --version" % executable)
+                print(subprocess.check_output([executable, "--version"]).decode())
+
+                archive.add(executable, basename(executable))
+        print("Packed {}.".format(archive_name(bdist)))
+
+
+@cli.command("upload")
+def upload():
+    """
+        Upload snapshot to snapshot server
+    """
+    subprocess.check_call(
+        [
+            "aws", "s3", "cp",
+            "--acl", "public-read",
+            DIST_DIR + "/",
+            "s3://snapshots.mitmproxy.org/%s/" % VERSION,
+            "--recursive",
+        ]
+    )
+
+
+if __name__ == "__main__":
+    cli()

release/known_hosts.enc

@@ -1 +0,0 @@
-gAAAAABaTif138dCP2-G3sAJxqh5icnwM0Zy7qh4HFCxeKQBMiVDr4nJyf9T82U677M_QKWRJmp_PsbnrshHXPylq0FuHwak7Yx7kdiLue6d85VQ7_kkMs-MlPM7_Xn54_zyuj1c0b3TVAuix2xHfFLdSd_mCxygFukLzf47OyYbno7lMY_-q0HZfVPz3PBZdk95wDcbYprmgEkVJZd64Tu_LG1JDDiz56LlqADMA4znMcSAoRmbVtHu-II09HMcX3TkmcqJsNv-IVHMs4fxW_DFsq9w5ARggL6ANMfhnFQPyMtgVHjGLkSjOMRshLkQUBVYx8yWEGaQOkP0doVtDS3fZ-MKc6OJC_NSs6gkm1rswjVsQsmgZGPIqjcVf9oCbFYcw0m-JrfB1irdsLoGzpfJaSGxveC7XqOd9ArBpCHFPVO-6ilu-E1qZelvL0HiplrFvJCMEev1U2YvznC1BWKpy81vJfH--64QKZ35yQBHMV_VoH-wi80EfWtz4ISvCMQWdjRAvhLHKHSYYhUSIgBZvCCQcPySdFpbDtwsQnzIqC8MQKG787w1FiYAwzdIHTWZuanENaPMALo0t0GgMSqPV4UUyw7dto8XSMqoUXOCuZNYjunVh7AzAKS7oMUYjDs38o92sWh5sZUpPfv2WYIiecTiQw4uPae7PdSwMhkI3WIOsSb8LURnG484vvgFc2jMpQThw-BHJx7tGYC0yFLouRH2O7m9x6xgiCiVA_u_BdOj_2PFufvOCaB9wno5Vo7C1hUERGWqoBZH0htBqxYci27hh8GFwkvj6OjFUyV_kk920cBYBDG4jS4bTrTzn_znJ9TNw2XkP98nA8cwlRYhDQG9FypJG0WwYkft3TVLSQ3Hq7t0nhvhSZvXts-3LR4S0_Hm0QgFUpUc-VHViinwK8_vQH3ZjvVlEWiXnzPdpAujjX_tQXsi13UE1Zp90wGeLrmdxGXq2K76Shytu8IwTcLNZ7m0jh8KmmfNwn6oZv-czqNmC4hh0OqRDFBrv3nnjDg2Vw74uKSZmXgtZlF_Zj9hPqxVWzj7lJUcyRqABBFbBH6lTSWPHLrzQ4eTex5dnOkXC8c3hRYDUt06xUkmDqaLK0rGFcfNXawZj1YqpUJW0qaNgbtBZRsSs92kblkETxCzcwxOfupmAhWdSkmCoxt019crodz3heREcyN2xcD9qHvdY49_FD3l3U6UhrWvmkDkzyLMd7VmRPWqlW0lkzrwav8e92leIq-xKFcvbnWgSdSCWWbXvIVJKcQ6hML3jX4oY7SoBs33U1Q0HfC7SuS5lqTASuRIOVCfIGeFfRwlIfEszbWg_WDoUjR6StaVq9tbtIC3mimWND82Z9r1NfUNxr8kFYIpH_6hbxhcW26HNBKr4wLxWFFE9l1QZORPM3s6z-lT4LzUPCkFExd_eYFx3X6yUJ3cHZhkQQzCLQqG7jQqvcMwDIfM-MXkJnttLfpBq0yiq0-mc-SEas5uy27iSJgbXnsV7G3YiKEelKW_uWP2bw-rQGG_AXMGNGF2A_aREsvGrEqPnyeHAxfS1bBcnqslpIzEwr9vyyJ5v_bxfHFQC4bwYMUvPGkjHVFc0Wrk7ss9P5Kd1bzh46H7OfroUbocmYBmHMMWEg-LvsG0RZil3KWh_CSyIIPETkDjuC3W7teT-wZK0zbTEaKCuz99Dg-tjzT6fP25ipoI70cX5R3KPwrLP3XNODRTsg_Jh7IpaXo9O3o8yLV9R6_rST_1KKJwzR2MMIXIvKaJQD9w2DZIaYx3tcVsXGCDnU4Tw2hhdB5wMCl3vHx83UHfjLxnc1tJ6ObpQUjwHM1SgHK8wLW409SVHphBbSjSilX5mIaR1S1SOTK53iFj5z6asZHY9JgDj11rng1uLKeirbrNZDnUme3NNYU-HX8Ret6oOesn3374uIHux1giqgR8VsPdkcMhvunx2oTP9R2fRBTSQ8sKNqDznRC8_qlQaRC94RnWO6VRNXVBT24cXq7HTepNp4f02UvUqQRyaIUmyn2S02mjLFECDm1iMxRhuacCKbI-WSKwJcm-7p39_Uh7m_nTl2VTseeQ-3NS6i-BiGmCHt3iDxR1Fkm31b50kWW3jCe6fcwMDeu3I_8mkQs_7mCFUjSDbvFUr2Y45a5guRlw63_KUW_mNN9td9hk8POWfxWEGhcZ9eRXh_eEdEaYZmviZdHi0I8pV52CqiEO-ZrnMw-w4rSpUQeRn9oKwp3GgB9j51RNlLqK9LTp-jfSGGi5GM-ab9sPgFCJLQ-HvHdGu0tQsF2wTD3qbJwNqapx28yNVfY6e8F2jOWjmP-zzFez8VNXcfoS--Ji_zI-VqsDx-cfz3DccWEjL6vjQOvaQTRwzhI7

release/rtool.pem

@@ -1,68 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-Proc-Type: 4,ENCRYPTED
-DEK-Info: AES-128-CBC,7A4E07094E1AC0B74BB3D172A73A62DB
-
-0H4otaU3/McUviuQh/6L1tQM/gUjAc0nn1QESZ/n2R/VOlmLGwYPlMxHLXcs6a+y
-byfQc/wcy1taPHSIVbEH1bYUfrFucLhlOq8jINPfsMqhPRGbH7DwTfAFNljZTGLU
-V9zMXedIGujOdbJ29gClbYkaNbHB7wd+7icpRMJ/buEC5W9lZXpt5kspsG9Ueu6Z
-T6rljxOQ5lyLun9E0y8H/f/RLViEyeEBSGK3UAHzz7+OJjKkKzVLfAFF2It4rfIh
-PdFBhfyp3ihfs4Dy+eG2LzVSZC+D01uj0WkhGwIGzpB4XAgNjtbUeaz9o8649lLl
-w4QuoX2cxbcLj4KVqSCml8aHA18oNtBdGx/s/+26bopmapcLwp01G51T/MMJTVpD
-P+6GoFvHxE7YKasJV2fxQW70U3B8Ruu5ImZkKa9Dw7VI1XL0DY/WvQ46tYZqllyt
-JeHxg21z0MeH6RiU7aT79tYB1KNcuuq9ujPFOSAED0C7TUEycvN4tyySPxqBAnBE
-yHT0UU3AE1VpQJaYDlQWrZPC813WoPI8pjLw+v99f918TWOOQkRNfobLqjgOe/Km
-cLT80vpaAjcwIZO9LTVYJ+gH/MDNLdgkIICAXL1MmgYBkhwuOw4+SxgfyK5gS4Pe
-hkJjoc3oxkFUA7uZ/X4xY0t+vwPfXcdp3oDgYMWfjTH3NenFaa78+7u/udEVrzJ8
-jtLDOM7XmJk+8EWu4KOiCct2pAhbYHAXzGx8X+5gZRNgPhBBDpg5I0+YERgy08nx
-QcBBBTxKEuOzgO+R1+9bucNzQYI61CRb+V4Sg2HjRsUP04J7TQDAeHZtpDbpIwey
-OKqXaf4874Mi3CYktbO5ZP9VxlTHdb6cagxkzLpnFwIkEBAKI5MOMxmYyK+7vwrL
-kn4dcNSReOLJaiLiMA9J3lswBPuVRRobnWCbLeHEq3a8j21JNZRBsZeKJCP1uPDR
-AZbnk4DxXZVlSaISGhpj7dpPqIuGnMP4FnMORSBxssdB9F6q03Bh2Vl59U184Lvc
-sbE5QvsSzJTQZc8790DTW90lwM3nfkJ+OePR4DwtTx1LdK8Z/mm6vll42Thwgsko
-SKm6n47xgnVBm6Kmbl8G248RAsRbG/f1TQYKBlU8iBKeFocW2LDCIuftzgUqjgcz
-5L/GlTbTzNPUT8VZ6WLtHBK6OBjebbE2zZ+W8mwlaVC+Z9Rush5Op4CO10Y1jK0P
-cwAsG6G16h5nxtGVz/C028BG1sJS+2XPel12+CrN2rrb6qP41YyLtWJVzhj5jFBk
-kAvf5g1w227Lt1CuRnICuPdP68MtzWuY5vQOE57eN/jVsuqmEHNwi8K0ZmtEwf5/
-Vq1QH/Nz/WP5vaIQGivtganAcTgdFD+HXmT0QS1qBot87w9LOU0JkMyBvaa5aXS9
-3QsVppnCn/x3OuE8SlujWvndJKb172u+0iR2mJ+OUuVMSjpfrSo5LQq03tmLOdD4
-mBg7WNuJOrjfyOq51S4zMrCqA2hoKL1lxE10RB+4RoHbpfpiblvlBVnY+cj6ic6t
-8MuiXtKFcV4KwujKgB0aS2smLSkaA8tXa4k3vhgIm68FZRGpCMuT6J+dEWiRgY8j
-FIWVlndlRustHN2qHyBipFm0Ei1iVK4qxcsLBhBI02ceXrcP5cFq8k3uuKJexKcf
-EPXPEDy5AJKxjIKdhEiPZmMyvSQuN1Nwu97b+QG685IfMpB/18WBYwFrTJOEU7pP
-a9uowuTaKXGugJIGhL6ziy5HN2zkKO2nZMp05/+nSXt4/j+C+pldG+rBNZF1SCPR
-AmB5YySQHUDey3SZGZhHrTdGN85M6A2PnfWzdsxoOSE9pfag8nx1AUm5CJhyQTH+
-m2OgUNALXJczOot0v0hHxL0XFHxXmMobbqCbtveufwaiRlDdihQeEXK3obIvEs3P
-bsb3OcKHPHFmmWluevX0wGsV9ZY+uSBvZaBRzmv5ij3n4CTpt+CVPn/IJq25E50Q
-QZ4nkGGQyX5yQtHhIGjagfKCFRrnpuwgmVegWLCSVx3Rev+blpLygpz77D447CCx
-uzAqTpAUiagv/U0av1B9U6fXwN5N8rN/qpbwQ/94nh+ELCuyBSEQHj6c9hIuy9pq
-yPU5hKY7InYcNkI9U2lS2ZtEZsm/NEzeG3oIBdoPC0NlFUjDeWZkEXi1Vt9rlk7K
-u3bOhqIVzxkeJIhde2D4CWMS9Kg8RFGhzB8nPOvCcU7vKukR5Ok/Q2bVtzPR3jRK
-MZrARlCJTtCnkWhkYQlCuFpafuzrItITA4M8L1ZwjcVcvJNamq9b/URQfbOfsDY0
-8Igu/zuGGnb0DecPHX9DUVws/tk1HXpIC6RFiZFNfH7Pb63TyqF1dc+9oHbqRqfq
-y05xekHY87Uzwyvn8Uch5j8dfoa3MvOWy/gEoK4ZMrsXyzywzLTHusaWnN172Oaz
-G3IljaBijzzwMGkEXmPiO+4WM/mp8dv29GpNj0/Dr8L/hYGcZ87CzWGnSu93yToi
-o1paUjM1zmK0CKHKda8FxaVUchIHEmi4DGxY1Ywj0NjeV5U3VQpLQawGm88aqdkq
-2aF5annHPNfp5tncL3eySrZxy6Yy9hI1/CCxV9r/31JZdRofqwftgKyxrTN0TzXL
-5fpvzZQly1B46yVzUUjPkez+t18Rr9i7Js/Dxop2+IHx1EDAlkeoz4OIDa6fUssp
-30eVjsy4QTV65HUS2vIraoqvKHwiabqz+QU/k6EVmln5cM6fuqN4m9HBiGVVLTac
-7WAzuW7D5mmIGXtl4JdoLXW9NuKAn7MIBixhlWY8/rdvBGRJ4zHQxM3RTmcAlaEA
-1/BoQUNXundypyFO/dYH01N4N4TNJDXTpOslmJHc78c09w4PN6/MjQMF8SkTHCJ8
-eKPXxQOh7arK/ibyhT6cVE29BmBS2OAeIYxRpqVOxndnSv+dpH02L5NjgDi97YIB
-bb2zMqyy+0ajQexov+PH1X2bxMLgY7GZ9zJ3GEyrq3MXYCLkR7IKm6R45xNMhFPO
-VWx0CLZCFIiHbdUg1YPWhaqHK5fqIAlQSDgdurnWvvz3Y9gnbZxuzMQt4sAwiN/+
-2XM5ekMi50JEtiUyIggtLJl/0HAiGeAEld9Gbtl1uTQjngC1i6fLzvvBwUCQQ175
------END RSA PRIVATE KEY-----
------BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAs0xeGx+bc+NCLGRJDr6y
-qv825685l1Tb6lhLvYVYUFJfcxL1V79ML13CO09TPxGTbmKw7iI33ef9N2mUHHVe
-24EjcSnYZ2Zx5zYIY4ejnDKCzM0md9XA3RpfTI9F3zGsJPt72veyHy6alCbF4NgX
-FgAvp2UJmWwTLnJj3oIEwfGJeEcQRKjCFjSJlHGxbB6L0MSCgRtDxwlBfinzkriZ
-0C8YyVv6u+DidHtobbZvwk/4uV3+KhoDfkpT2VE57a2KXr4c9Fekzr6szduFMkcI
-KPqKouOOnaiNjOoKofbZ5I6cB6qoMkh+ebJvfPuEYBwhHmDDhpdjdk5v3LkeD5kt
-0GtOaF5A3LbdZhgsC+1v1dHrL6Vndrb9KhO8ATHvlzJOonV73AoJFygp6upCvIt+
-tPUuy3VE0kjc2mJDDJr5V28pSBJ5r9uz1rXZWP+9aduR3M+RUGzHS7cUdsl+lHyQ
-TFQAQIDhWXyokMGqVZBm9xJ2KcDiQAEr5+kP0BrhPmBtfHSnLA1tIXlFcOqakvyQ
-q1cACMvwbkZpupTqMhCr2S4V9dKxnlO5EIWYJmme9moU/zu1KA0mcwqfnNeDOR2J
-6c3/x3E+AqBoFzDQb6LQ2Iw+9bkKWKgDml4paLPu/ot8vA2ZB9/mujFUvp/1MLuD
-5I1jQDXFbjAgAnaSyqyW368CAwEAAQ==
------END PUBLIC KEY-----

release/rtool.py

@@ -8,14 +8,12 @@ import platform
 import re
 import runpy
 import shlex
-import shutil
 import subprocess
 import tarfile
 import zipfile
-from os.path import join, abspath, dirname, exists, basename
+from os.path import join, abspath, dirname
 
 import click
-import cryptography.fernet
 import pysftp
 
 # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
@@ -86,18 +84,6 @@ def get_version(dev: bool = False, build: bool = False) -> str:
     return x["get_version"](dev, build, True)
 
 
-def set_version(dev: bool) -> None:
-    """
-    Update version information in mitmproxy's version.py to either include hardcoded information or not.
-    """
-    version = get_version(dev, dev)
-    with open(VERSION_FILE, "r") as f:
-        content = f.read()
-    content = re.sub(r'^VERSION = ".+?"', 'VERSION = "{}"'.format(version), content, flags=re.M)
-    with open(VERSION_FILE, "w") as f:
-        f.write(content)
-
-
 def archive_name(bdist: str) -> str:
     if platform.system() == "Windows":
         ext = "zip"
@@ -146,24 +132,6 @@ def cli():
     pass
 
 
-@cli.command("encrypt")
-@click.argument('infile', type=click.File('rb'))
-@click.argument('outfile', type=click.File('wb'))
-@click.argument('key', envvar='RTOOL_KEY')
-def encrypt(infile, outfile, key):
-    f = cryptography.fernet.Fernet(key.encode())
-    outfile.write(f.encrypt(infile.read()))
-
-
-@cli.command("decrypt")
-@click.argument('infile', type=click.File('rb'))
-@click.argument('outfile', type=click.File('wb'))
-@click.argument('key', envvar='RTOOL_KEY')
-def decrypt(infile, outfile, key):
-    f = cryptography.fernet.Fernet(key.encode())
-    outfile.write(f.decrypt(infile.read()))
-
-
 @cli.command("contributors")
 def contributors():
     """
@@ -176,97 +144,6 @@ def contributors():
             f.write(contributors_data.encode())
 
 
-@cli.command("wheel")
-def make_wheel():
-    """
-    Build a Python wheel
-    """
-    set_version(True)
-    try:
-        subprocess.check_call([
-            "tox", "-e", "wheel",
-        ], env={
-            **os.environ,
-            "VERSION": get_version(True),
-        })
-    finally:
-        set_version(False)
-
-
-@cli.command("bdist")
-def make_bdist():
-    """
-    Build a binary distribution
-    """
-    if exists(PYINSTALLER_TEMP):
-        shutil.rmtree(PYINSTALLER_TEMP)
-    if exists(PYINSTALLER_DIST):
-        shutil.rmtree(PYINSTALLER_DIST)
-
-    os.makedirs(DIST_DIR, exist_ok=True)
-
-    for bdist, tools in sorted(BDISTS.items()):
-        with Archive(join(DIST_DIR, archive_name(bdist))) as archive:
-            for tool in tools:
-                # We can't have a folder and a file with the same name.
-                if tool == "mitmproxy":
-                    tool = "mitmproxy_main"
-                # This is PyInstaller, so it messes up paths.
-                # We need to make sure that we are in the spec folder.
-                with chdir(PYINSTALLER_SPEC):
-                    print("Building %s binary..." % tool)
-                    excludes = []
-                    if tool != "mitmweb":
-                        excludes.append("mitmproxy.tools.web")
-                    if tool != "mitmproxy_main":
-                        excludes.append("mitmproxy.tools.console")
-
-                    # Overwrite mitmproxy/version.py to include commit info
-                    set_version(True)
-                    try:
-                        subprocess.check_call(
-                            [
-                                "pyinstaller",
-                                "--clean",
-                                "--workpath", PYINSTALLER_TEMP,
-                                "--distpath", PYINSTALLER_DIST,
-                                "--additional-hooks-dir", PYINSTALLER_HOOKS,
-                                "--onefile",
-                                "--console",
-                                "--icon", "icon.ico",
-                                # This is PyInstaller, so setting a
-                                # different log level obviously breaks it :-)
-                                # "--log-level", "WARN",
-                            ]
-                            + [x for e in excludes for x in ["--exclude-module", e]]
-                            + PYINSTALLER_ARGS
-                            + [tool]
-                        )
-                    finally:
-                        set_version(False)
-                    # Delete the spec file - we're good without.
-                    os.remove("{}.spec".format(tool))
-
-                # Test if it works at all O:-)
-                executable = join(PYINSTALLER_DIST, tool)
-                if platform.system() == "Windows":
-                    executable += ".exe"
-
-                # Remove _main suffix from mitmproxy executable
-                if "_main" in executable:
-                    shutil.move(
-                        executable,
-                        executable.replace("_main", "")
-                    )
-                    executable = executable.replace("_main", "")
-
-                print("> %s --version" % executable)
-                print(subprocess.check_output([executable, "--version"]).decode())
-
-                archive.add(executable, basename(executable))
-        print("Packed {}.".format(archive_name(bdist)))
-
-
 @cli.command("upload-release")
 @click.option('--username', prompt=True)
 @click.password_option(confirmation_prompt=False)
@@ -305,78 +182,5 @@ def homebrew_pr():
     ])
 
 
-@cli.command("upload-snapshot")
-@click.option("--host", envvar="SNAPSHOT_HOST", prompt=True)
-@click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22)
-@click.option("--user", envvar="SNAPSHOT_USER", prompt=True)
-@click.option("--private-key", default=join(RELEASE_DIR, "rtool.pem"))
-@click.option("--private-key-password", envvar="SNAPSHOT_PASS", prompt=True, hide_input=True)
-@click.option("--wheel/--no-wheel", default=False)
-@click.option("--bdist/--no-bdist", default=False)
-@click.option("--installer/--no-installer", default=False)
-def upload_snapshot(host, port, user, private_key, private_key_password, wheel, bdist, installer):
-    """
-    Upload snapshot to snapshot server
-    """
-    cnopts = pysftp.CnOpts(
-        knownhosts=join(RELEASE_DIR, 'known_hosts')
-    )
-    with pysftp.Connection(host=host,
-                           port=port,
-                           username=user,
-                           private_key=private_key,
-                           private_key_pass=private_key_password,
-                           cnopts=cnopts) as sftp:
-        dir_name = "snapshots/v{}".format(get_version())
-        sftp.makedirs(dir_name)
-        with sftp.cd(dir_name):
-            files = []
-            if wheel:
-                files.append(wheel_name())
-            if bdist:
-                for bdist in sorted(BDISTS.keys()):
-                    files.append(archive_name(bdist))
-            if installer:
-                files.append(installer_name())
-
-            for f in files:
-                local_path = join(DIST_DIR, f)
-                remote_filename = re.sub(
-                    r"{version}(\.dev\d+(-0x[0-9a-f]+)?)?".format(version=get_version()),
-                    get_version(True, True),
-                    f
-                )
-                symlink_path = "../{}".format(f.replace(get_version(), "latest"))
-
-                # Upload new version
-                print("Uploading {} as {}...".format(f, remote_filename))
-                with click.progressbar(length=os.stat(local_path).st_size) as bar:
-                    # We hide the file during upload
-                    sftp.put(
-                        local_path,
-                        "." + remote_filename,
-                        callback=lambda done, total: bar.update(done - bar.pos)
-                    )
-
-                # Delete old versions
-                old_version = f.replace(get_version(), "*")
-                for f_old in sftp.listdir():
-                    if fnmatch.fnmatch(f_old, old_version):
-                        print("Removing {}...".format(f_old))
-                        sftp.remove(f_old)
-
-                # Show new version
-                sftp.rename("." + remote_filename, remote_filename)
-
-                # update symlink for the latest release
-                if sftp.lexists(symlink_path):
-                    print("Removing {}...".format(symlink_path))
-                    sftp.remove(symlink_path)
-                if f != wheel_name():
-                    # "latest" isn't a proper wheel version, so this could not be installed.
-                    # https://github.com/mitmproxy/mitmproxy/issues/1065
-                    sftp.symlink("v{}/{}".format(get_version(), remote_filename), symlink_path)
-
-
 if __name__ == "__main__":
     cli()

tox.ini

@@ -45,14 +45,12 @@ commands =
   pathod --version
   pathoc --version
 
-[testenv:rtool]
-passenv = SKIP_MITMPROXY SNAPSHOT_HOST SNAPSHOT_PORT SNAPSHOT_USER SNAPSHOT_PASS RTOOL_KEY
+[testenv:cibuild]
+passenv = TRAVIS_TAG TRAVIS_BRANCH AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
 deps =
   -rrequirements.txt
   pyinstaller==3.3.1
-  twine==1.9.1
-  pysftp==0.2.9
-
+  awscli
 commands =
   mitmdump --version
-  python ./release/rtool.py {posargs}
+  python ./release/ci.py {posargs}