Windows: build pyinstaller onedir and use it for installer

This greatly improves startup time as pyinstaller doesn't have to
unpack everything on startup. The same also applies to macOS and Linux,
but there we 1) don't have installers to hide all the files and
2) have a filesystem that deals much better with lots of small files.

Additionally, simplify cibuild to be a bit more reasonable.

release/cibuild.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python3
 
 import contextlib
-import glob
 import os
-import pathlib
 import platform
 import re
 import shutil
@@ -12,15 +10,17 @@ import sys
 import tarfile
 import urllib.request
 import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional, Union
 
 import click
-import parver
-
 import cryptography.fernet
+import parver
 
 
 @contextlib.contextmanager
-def chdir(path: str):  # pragma: no cover
+def chdir(path: Path):  # pragma: no cover
     old_dir = os.getcwd()
     os.chdir(path)
     yield
@@ -39,6 +39,16 @@ def bool_from_env(envvar: str) -> bool:
         return True
 
 
+class ZipFile2(zipfile.ZipFile):
+    # ZipFile and tarfile have slightly different APIs. Let's fix that.
+    def add(self, name: str, arcname: str) -> None:
+        return self.write(name, arcname)
+
+    def __enter__(self) -> "ZipFile2":
+        return self
+
+
+@dataclass(frozen=True, repr=False)
 class BuildEnviron:
     PLATFORM_TAGS = {
         "Darwin": "osx",
@@ -46,100 +56,89 @@ class BuildEnviron:
         "Linux": "linux",
     }
 
-    def __init__(
+    system: str
-            self,
+    root_dir: Path
-            *,
+    branch: Optional[str] = None
-            system="",
+    tag: Optional[str] = None
-            root_dir="",
+    is_pull_request: bool = True
-            github_ref="",
+    should_build_wheel: bool = False
-            github_event_name="",
+    should_build_docker: bool = False
-            should_build_wheel=False,
+    should_build_pyinstaller: bool = False
-            should_build_docker=False,
+    should_build_wininstaller: bool = False
-            should_build_pyinstaller=False,
+    has_aws_creds: bool = False
-            should_build_wininstaller=False,
+    has_twine_creds: bool = False
-            has_aws_creds=False,
+    docker_username: Optional[str] = None
-            has_twine_creds=False,
+    docker_password: Optional[str] = None
-            docker_username="",
+    build_key: Optional[str] = None
-            docker_password="",
-            build_key="",
-    ):
-        self.system = system
-        self.root_dir = root_dir
-
-        self.should_build_wheel = should_build_wheel
-        self.should_build_docker = should_build_docker
-        self.should_build_pyinstaller = should_build_pyinstaller
-        self.should_build_wininstaller = should_build_wininstaller
-
-        self.github_ref = github_ref
-        self.github_event_name = github_event_name
-
-        self.has_aws_creds = has_aws_creds
-        self.has_twine_creds = has_twine_creds
-        self.docker_username = docker_username
-        self.docker_password = docker_password
-        self.build_key = build_key
 
     @classmethod
-    def from_env(cls):
+    def from_env(cls) -> "BuildEnviron":
+        branch = None
+        tag = None
+
+        if ref := os.environ.get("GITHUB_REF", ""):
+            if ref.startswith("refs/heads/"):
+                branch = ref.replace("refs/heads/", "")
+            if ref.startswith("refs/pull/"):
+                branch = "pr-" + ref.split("/")[2]
+            if ref.startswith("refs/tags/"):
+                tag = ref.replace("refs/tags/", "")
+
+        is_pull_request = os.environ.get("GITHUB_EVENT_NAME", "pull_request") == "pull_request"
+
+        is_untrusted = (
+                is_pull_request
+                or
+                os.environ.get("GITHUB_REPOSITORY", "other") != "mitmproxy/mitmproxy"
+        )
+        has_creds = any(
+            x in os.environ for x in
+            ["AWS_ACCESS_KEY_ID",
+             "TWINE_USERNAME", "TWINE_PASSWORD",
+             "DOCKER_USERNAME", "DOCKER_PASSWORD",
+             "CI_BUILD_KEY"]
+        )
+        if is_untrusted and has_creds:
+            raise RuntimeError("Found upload credentials even though we aren't running on CI!")
+
         return cls(
             system=platform.system(),
-            root_dir=os.path.normpath(os.path.join(os.path.dirname(__file__), "..")),
+            root_dir=Path(__file__).parent.parent,
-            github_ref=os.environ.get("GITHUB_REF", ""),
+            branch=branch,
-            github_event_name=os.environ.get("GITHUB_EVENT_NAME", ""),
+            tag=tag,
+            is_pull_request=is_pull_request,
             should_build_wheel=bool_from_env("CI_BUILD_WHEEL"),
             should_build_pyinstaller=bool_from_env("CI_BUILD_PYINSTALLER"),
             should_build_wininstaller=bool_from_env("CI_BUILD_WININSTALLER"),
             should_build_docker=bool_from_env("CI_BUILD_DOCKER"),
             has_aws_creds=bool_from_env("AWS_ACCESS_KEY_ID"),
             has_twine_creds=bool_from_env("TWINE_USERNAME") and bool_from_env("TWINE_PASSWORD"),
-            docker_username=os.environ.get("DOCKER_USERNAME", ""),
+            docker_username=os.environ.get("DOCKER_USERNAME", None),
-            docker_password=os.environ.get("DOCKER_PASSWORD", ""),
+            docker_password=os.environ.get("DOCKER_PASSWORD", None),
-            build_key=os.environ.get("CI_BUILD_KEY", ""),
+            build_key=os.environ.get("CI_BUILD_KEY", None),
         )
 
-    def archive(self, path):
+    def archive(self, path: Path) -> Union[tarfile.TarFile, ZipFile2]:
-        # ZipFile and tarfile have slightly different APIs. Fix that.
         if self.system == "Windows":
-            a = zipfile.ZipFile(path, "w")
+            return ZipFile2(path, "w")
-            a.add = a.write
-            return a
         else:
             return tarfile.open(path, "w:gz")
 
-    def archive_name(self, bdist: str) -> str:
+    @property
+    def archive_path(self) -> Path:
         if self.system == "Windows":
             ext = "zip"
         else:
             ext = "tar.gz"
-        return "{project}-{version}-{platform}.{ext}".format(
+        return self.dist_dir / f"mitmproxy-{self.version}-{self.platform_tag}.{ext}"
-            project=bdist,
-            version=self.version,
-            platform=self.platform_tag,
-            ext=ext
-        )
 
     @property
-    def bdists(self):
+    def build_dir(self) -> Path:
-        return {
+        return self.release_dir / "build"
-            "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
-        }
 
     @property
-    def branch(self) -> str:
+    def dist_dir(self) -> Path:
-        if self.github_ref and self.github_ref.startswith("refs/heads/"):
+        return self.release_dir / "dist"
-            return self.github_ref.replace("refs/heads/", "")
-        if self.github_ref and self.github_ref.startswith("refs/pull/"):
-            return "pr-" + self.github_ref.split("/")[2]
-        return ""
-
-    @property
-    def build_dir(self) -> str:
-        return os.path.join(self.release_dir, "build")
-
-    @property
-    def dist_dir(self) -> str:
-        return os.path.join(self.release_dir, "dist")
 
     @property
     def docker_tag(self) -> str:
@@ -159,7 +158,6 @@ class BuildEnviron:
             "release_dir",
             "build_dir",
             "dist_dir",
-            "bdists",
             "upload_dir",
             "should_build_wheel",
             "should_build_pyinstaller",
@@ -177,8 +175,7 @@ class BuildEnviron:
         Check that version numbers match our conventions.
         Raises a ValueError if there is a mismatch.
         """
-        with open(pathlib.Path(self.root_dir) / "mitmproxy" / "version.py") as f:
+        contents = (self.root_dir / "mitmproxy" / "version.py").read_text("utf8")
-            contents = f.read()
         match = re.search(r'^VERSION = "(.+?)"', contents, re.M)
         assert match
         version = match.group(1)
@@ -211,7 +208,7 @@ class BuildEnviron:
 
     @property
     def is_prod_release(self) -> bool:
-        if not (self.tag and self.tag.startswith("v")):
+        if not self.tag or not self.tag.startswith("v"):
             return False
         try:
             v = parver.Version.parse(self.version, strict=True)
@@ -219,21 +216,15 @@ class BuildEnviron:
             return False
         return not v.is_prerelease
 
-    @property
-    def is_pull_request(self) -> bool:
-        if self.github_event_name == "pull_request":
-            return True
-        return False
-
     @property
     def platform_tag(self) -> str:
         if self.system in self.PLATFORM_TAGS:
             return self.PLATFORM_TAGS[self.system]
-        raise BuildError("Unsupported platform: %s" % self.system)
+        raise BuildError(f"Unsupported platform: {self.system}")
 
     @property
-    def release_dir(self) -> str:
+    def release_dir(self) -> Path:
-        return os.path.join(self.root_dir, "release")
+        return self.root_dir / "release"
 
     @property
     def should_upload_docker(self) -> bool:
@@ -258,18 +249,12 @@ class BuildEnviron:
             self.has_twine_creds,
         ])
 
-    @property
-    def tag(self) -> str:
-        if self.github_ref and self.github_ref.startswith("refs/tags/"):
-            return self.github_ref.replace("refs/tags/", "")
-        return ""
-
     @property
     def upload_dir(self) -> str:
         if self.tag:
             return self.version
         else:
-            return "branches/%s" % self.version
+            return f"branches/{self.version}"
 
     @property
     def version(self) -> str:
@@ -287,7 +272,7 @@ class BuildEnviron:
             raise BuildError("We're on neither a tag nor a branch - could not establish version")
 
 
-def build_wheel(be: BuildEnviron):  # pragma: no cover
+def build_wheel(be: BuildEnviron) -> None:  # pragma: no cover
     click.echo("Building wheel...")
     subprocess.check_call([
         "python",
@@ -296,22 +281,21 @@ def build_wheel(be: BuildEnviron):  # pragma: no cover
         "bdist_wheel",
         "--dist-dir", be.dist_dir,
     ])
-    whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))
+    whl, = be.dist_dir.glob('mitmproxy-*-py3-none-any.whl')
     click.echo(f"Found wheel package: {whl}")
     subprocess.check_call(["tox", "-e", "wheeltest", "--", whl])
-    return whl
 
 
-def build_docker_image(be: BuildEnviron):  # pragma: no cover
+def build_docker_image(be: BuildEnviron) -> None:  # pragma: no cover
-    whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))
+    whl, = be.dist_dir.glob('mitmproxy-*-py3-none-any.whl')
-    whl = str(pathlib.Path(whl).relative_to(pathlib.Path(".").absolute()))
+    whl = whl.relative_to(Path(".").absolute())
     click.echo("Building Docker images...")
     subprocess.check_call([
         "docker",
         "build",
         "--tag", be.docker_tag,
         "--build-arg", f"WHEEL_MITMPROXY={whl}",
-        "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(os.path.basename(whl)),
+        "--build-arg", f"WHEEL_BASENAME_MITMPROXY={whl.name}",
         "--file", "release/docker/Dockerfile",
         "."
     ])
@@ -328,101 +312,100 @@ def build_docker_image(be: BuildEnviron):  # pragma: no cover
     assert "Mitmproxy: " in r.stdout.decode()
 
 
-def build_pyinstaller(be: BuildEnviron):  # pragma: no cover
+def build_pyinstaller(be: BuildEnviron) -> None:  # pragma: no cover
     click.echo("Building pyinstaller package...")
 
-    PYINSTALLER_SPEC = os.path.join(be.release_dir, "specs")
+    PYINSTALLER_SPEC = be.release_dir / "specs"
-    # PyInstaller 3.2 does not bundle pydivert's Windivert binaries
+    PYINSTALLER_HOOKS = be.release_dir / "hooks"
-    PYINSTALLER_HOOKS = os.path.abspath(os.path.join(be.release_dir, "hooks"))
+    PYINSTALLER_TEMP = be.build_dir / "pyinstaller"
-    PYINSTALLER_TEMP = os.path.abspath(os.path.join(be.build_dir, "pyinstaller"))
+    PYINSTALLER_DIST = be.build_dir / "binaries" / be.platform_tag
-    PYINSTALLER_DIST = os.path.abspath(os.path.join(be.build_dir, "binaries", be.platform_tag))
 
-    # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
+    if PYINSTALLER_TEMP.exists():
-    # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
-    if platform.system() == "Windows":
-        PYINSTALLER_ARGS = [
-            # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
-            "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
-        ]
-    else:
-        PYINSTALLER_ARGS = []
-
-    if os.path.exists(PYINSTALLER_TEMP):
         shutil.rmtree(PYINSTALLER_TEMP)
-    if os.path.exists(PYINSTALLER_DIST):
+    if PYINSTALLER_DIST.exists():
         shutil.rmtree(PYINSTALLER_DIST)
 
-    for bdist, tools in sorted(be.bdists.items()):
+    if be.platform_tag == "windows":
-        with be.archive(os.path.join(be.dist_dir, be.archive_name(bdist))) as archive:
+        with chdir(PYINSTALLER_SPEC):
-            for tool in tools:
+            click.echo("Building PyInstaller binaries in directory mode...")
-                # We can't have a folder and a file with the same name.
+            subprocess.check_call(
-                if tool == "mitmproxy":
+                [
-                    tool = "mitmproxy_main"
+                    "pyinstaller",
-                # This is PyInstaller, so it messes up paths.
+                    "--clean",
-                # We need to make sure that we are in the spec folder.
+                    "--workpath", PYINSTALLER_TEMP,
-                with chdir(PYINSTALLER_SPEC):
+                    "--distpath", PYINSTALLER_DIST,
-                    click.echo("Building PyInstaller %s binary..." % tool)
+                    "./windows-dir.spec"
-                    excludes = []
+                ]
-                    if tool != "mitmweb":
+            )
-                        excludes.append("mitmproxy.tools.web")
+            for tool in ["mitmproxy", "mitmdump", "mitmweb"]:
-                    if tool != "mitmproxy_main":
+                click.echo(f"> {tool} --version")
-                        excludes.append("mitmproxy.tools.console")
+                executable = (PYINSTALLER_DIST / "onedir" / tool).with_suffix(".exe")
-
-                    subprocess.check_call(
-                        [
-                            "pyinstaller",
-                            "--clean",
-                            "--workpath", PYINSTALLER_TEMP,
-                            "--distpath", PYINSTALLER_DIST,
-                            "--additional-hooks-dir", PYINSTALLER_HOOKS,
-                            "--onefile",
-                            "--console",
-                            "--icon", "icon.ico",
-                            # This is PyInstaller, so setting a
-                            # different log level obviously breaks it :-)
-                            # "--log-level", "WARN",
-                        ]
-                        + [x for e in excludes for x in ["--exclude-module", e]]
-                        + PYINSTALLER_ARGS
-                        + [tool]
-                    )
-                    # Delete the spec file - we're good without.
-                    os.remove(f"{tool}.spec")
-
-                # Test if it works at all O:-)
-                executable = os.path.join(PYINSTALLER_DIST, tool)
-                if platform.system() == "Windows":
-                    executable += ".exe"
-
-                # Remove _main suffix from mitmproxy executable
-                if "_main" in executable:
-                    shutil.move(
-                        executable,
-                        executable.replace("_main", "")
-                    )
-                    executable = executable.replace("_main", "")
-
-                click.echo("> %s --version" % executable)
                 click.echo(subprocess.check_output([executable, "--version"]).decode())
 
-                archive.add(executable, os.path.basename(executable))
+    with be.archive(be.archive_path) as archive:
-        click.echo("Packed {}.".format(be.archive_name(bdist)))
+        for tool in ["mitmproxy", "mitmdump", "mitmweb"]:
+            # We can't have a folder and a file with the same name.
+            if tool == "mitmproxy":
+                tool = "mitmproxy_main"
+            # Make sure that we are in the spec folder.
+            with chdir(PYINSTALLER_SPEC):
+                click.echo(f"Building PyInstaller {tool} binary...")
+                excludes = []
+                if tool != "mitmweb":
+                    excludes.append("mitmproxy.tools.web")
+                if tool != "mitmproxy_main":
+                    excludes.append("mitmproxy.tools.console")
+
+                subprocess.check_call(
+                    [   # type: ignore
+                        "pyinstaller",
+                        "--clean",
+                        "--workpath", PYINSTALLER_TEMP,
+                        "--distpath", PYINSTALLER_DIST,
+                        "--additional-hooks-dir", PYINSTALLER_HOOKS,
+                        "--onefile",
+                        "--console",
+                        "--icon", "icon.ico",
+                    ]
+                    + [x for e in excludes for x in ["--exclude-module", e]]
+                    + [tool]
+                )
+                # Delete the spec file - we're good without.
+                os.remove(f"{tool}.spec")
+
+            executable = PYINSTALLER_DIST / tool
+            if be.platform_tag == "windows":
+                executable = executable.with_suffix(".exe")
+
+            # Remove _main suffix from mitmproxy executable
+            if "_main" in executable.name:
+                executable = executable.rename(
+                    executable.with_name(executable.name.replace("_main", ""))
+                )
+
+            # Test if it works at all O:-)
+            click.echo(f"> {executable} --version")
+            click.echo(subprocess.check_output([executable, "--version"]).decode())
+
+            archive.add(str(executable), str(executable.name))
+    click.echo("Packed {}.".format(be.archive_path.name))
 
 
-def build_wininstaller(be: BuildEnviron):  # pragma: no cover
+def build_wininstaller(be: BuildEnviron) -> None:  # pragma: no cover
-    if not be.build_key:
-        click.echo("Cannot build windows installer without secret key.")
-        return
     click.echo("Building wininstaller package...")
 
-    IB_VERSION = "20.9.0"
+    IB_VERSION = "20.12.0"
-    IB_DIR = pathlib.Path(be.release_dir) / "installbuilder"
+    IB_DIR = be.release_dir / "installbuilder"
     IB_SETUP = IB_DIR / "setup" / f"{IB_VERSION}-installer.exe"
-    IB_CLI = fr"C:\Program Files (x86)\VMware InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe"
+    IB_CLI = Path(fr"C:\Program Files\VMware InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe")
     IB_LICENSE = IB_DIR / "license.xml"
 
-    if not os.path.isfile(IB_CLI):
+    if not IB_LICENSE.exists() and not be.build_key:
-        if not os.path.isfile(IB_SETUP):
+        click.echo("Cannot build windows installer without secret key.")
+        return
+
+    if not IB_CLI.exists():
+        if not IB_SETUP.exists():
             click.echo("Downloading InstallBuilder...")
 
             def report(block, blocksize, total):
@@ -430,22 +413,25 @@ def build_wininstaller(be: BuildEnviron):  # pragma: no cover
                 if round(100 * done / total) != round(100 * (done - blocksize) / total):
                     click.secho(f"Downloading... {round(100 * done / total)}%")
 
+            tmp = IB_SETUP.with_suffix(".tmp")
             urllib.request.urlretrieve(
-                f"https://clients.bitrock.com/installbuilder/installbuilder-enterprise-{IB_VERSION}-windows-installer.exe",
+                f"https://clients.bitrock.com/installbuilder/installbuilder-enterprise-{IB_VERSION}-windows-x64-installer.exe",
-                IB_SETUP.with_suffix(".tmp"),
+                tmp,
                 reporthook=report
             )
-            shutil.move(str(IB_SETUP.with_suffix(".tmp")), str(IB_SETUP))
+            tmp.rename(IB_SETUP)
 
         click.echo("Install InstallBuilder...")
-        subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"], check=True)
+        subprocess.run([IB_SETUP, "--mode", "unattended", "--unattendedmodeui", "none"], check=True)
-        assert os.path.isfile(IB_CLI)
+        assert IB_CLI.is_file()
 
-    click.echo("Decrypt InstallBuilder license...")
+    if not IB_LICENSE.exists():
-    f = cryptography.fernet.Fernet(be.build_key.encode())
+        assert be.build_key
-    with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, \
+        click.echo("Decrypt InstallBuilder license...")
-            open(IB_LICENSE, "wb") as outfile:
+        f = cryptography.fernet.Fernet(be.build_key.encode())
-        outfile.write(f.decrypt(infile.read()))
+        with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, \
+                open(IB_LICENSE, "wb") as outfile:
+            outfile.write(f.decrypt(infile.read()))
 
     click.echo("Run InstallBuilder...")
     subprocess.run([
@@ -457,8 +443,7 @@ def build_wininstaller(be: BuildEnviron):  # pragma: no cover
         "--setvars", f"project.version={be.version}",
         "--verbose"
     ], check=True)
-    assert os.path.isfile(
+    assert (be.dist_dir / f"mitmproxy-{be.version}-windows-installer.exe").exists()
-        os.path.join(be.dist_dir, f"mitmproxy-{be.version}-windows-installer.exe"))
 
 
 @click.group(chain=True)
@@ -507,18 +492,18 @@ def upload():  # pragma: no cover
         return
 
     if be.should_upload_aws:
-        num_files = len([name for name in os.listdir(be.dist_dir) if os.path.isfile(name)])
+        num_files = len([name for name in be.dist_dir.iterdir() if name.is_file()])
         click.echo(f"Uploading {num_files} files to AWS dir {be.upload_dir}...")
         subprocess.check_call([
             "aws", "s3", "cp",
             "--acl", "public-read",
-            be.dist_dir + "/",
+            f"{be.dist_dir}/",
             f"s3://snapshots.mitmproxy.org/{be.upload_dir}/",
             "--recursive",
         ])
 
     if be.should_upload_pypi:
-        whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
+        whl, = be.dist_dir.glob('mitmproxy-*-py3-none-any.whl')
         click.echo(f"Uploading {whl} to PyPi...")
         subprocess.check_call(["twine", "upload", whl])
 

release/installbuilder/mitmproxy.xml

@@ -31,7 +31,7 @@
                     <distributionFileList>
                         <distributionFile>
                             <allowWildcards>1</allowWildcards>
-                            <origin>../build/binaries/${platform_name}/*</origin>
+                            <origin>../build/binaries/windows/onedir/*</origin>
                         </distributionFile>
                         <distributionFile>
                             <origin>run.ps1</origin>

release/installbuilder/run.ps1

@@ -1,5 +1,6 @@
+$tool = $args[0]
 if (Get-Command wt -ErrorAction SilentlyContinue) {
-	Start-Process wt -ArgumentList "powershell.exe","-NoExit","-Command",$args[0]
+	Start-Process wt -ArgumentList "powershell.exe","-Command","& '$PSScriptRoot\$tool.exe'"
 } else { 
-	Start-Process powershell -ArgumentList "-NoExit","-Command",$args[0]
+	Start-Process powershell -ArgumentList "-Command","& '$PSScriptRoot\$tool.exe'"
 }

release/specs/mitmproxy

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from mitmproxy.tools.main import mitmproxy
+mitmproxy()

release/specs/windows-dir.spec

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+from PyInstaller.building.api import PYZ, EXE, COLLECT
+from PyInstaller.building.build_main import Analysis
+
+assert SPECPATH == "."
+
+here = Path(r".")
+tools = ["mitmproxy", "mitmdump", "mitmweb"]
+
+analysis = Analysis(
+    tools,
+    excludes=["tcl", "tk", "tkinter"],
+    pathex=[str(here)],
+    hookspath=[str(here / ".." / "hooks")],
+)
+
+pyz = PYZ(analysis.pure, analysis.zipped_data)
+executables = []
+for tool in tools:
+    executables.append(EXE(
+        pyz,
+        # analysis.scripts has all runtime hooks and all of our tools.
+        # remove the other tools.
+        [s for s in analysis.scripts if s[0] not in tools or s[0] == tool],
+        [],
+        exclude_binaries=True,
+        name=tool,
+        console=True,
+        upx=False,
+        icon='icon.ico'
+    ))
+
+COLLECT(
+    *executables,
+    analysis.binaries,
+    analysis.zipfiles,
+    analysis.datas,
+    strip=False,
+    upx=False,
+    name="onedir"
+)

test/release/test_cibuild.py

@@ -1,10 +1,12 @@
 import io
-import os
+from pathlib import Path
 
 import pytest
 
 from release import cibuild
 
+root = Path(__file__).parent.parent.parent
+
 
 def test_buildenviron_live():
     be = cibuild.BuildEnviron.from_env()
@@ -14,13 +16,12 @@ def test_buildenviron_live():
 def test_buildenviron_common():
     be = cibuild.BuildEnviron(
         system="Linux",
-        root_dir="/foo",
+        root_dir=root,
-        github_ref="refs/heads/master",
+        branch="master",
     )
-    assert be.release_dir == os.path.join(be.root_dir, "release")
+    assert be.release_dir == be.root_dir / "release"
-    assert be.dist_dir == os.path.join(be.root_dir, "release", "dist")
+    assert be.dist_dir == be.root_dir / "release" / "dist"
-    assert be.build_dir == os.path.join(be.root_dir, "release", "build")
+    assert be.build_dir == be.root_dir / "release" / "build"
-    assert be.is_pull_request is False
     assert not be.has_docker_creds
 
     cs = io.StringIO()
@@ -29,7 +30,7 @@ def test_buildenviron_common():
 
     be = cibuild.BuildEnviron(
         system="Unknown",
-        root_dir="/foo",
+        root_dir=root,
     )
     with pytest.raises(cibuild.BuildError):
         be.version
@@ -37,41 +38,27 @@ def test_buildenviron_common():
         be.platform_tag
 
 
-def test_buildenviron_pr():
+def test_buildenviron_pr(monkeypatch):
     # Simulates a PR. We build everything, but don't have access to secret
     # credential env variables.
-    be = cibuild.BuildEnviron(
+    monkeypatch.setenv("GITHUB_REF", "refs/pull/42/merge")
-        github_event_name="pull_request",
+    monkeypatch.setenv("CI_BUILD_WHEEL", "1")
-        should_build_wheel=True,
+    monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request")
-        should_build_pyinstaller=True,
+
-        should_build_docker=True,
+    be = cibuild.BuildEnviron.from_env()
-    )
+    assert be.branch == "pr-42"
     assert be.is_pull_request
-
+    assert be.should_build_wheel
-
+    assert not be.should_upload_pypi
-def test_ci_systems():
-    github = cibuild.BuildEnviron(
-        github_event_name="pull_request",
-        github_ref="refs/heads/master"
-    )
-    assert github.is_pull_request
-    assert github.branch == "master"
-    assert github.tag == ""
-
-    github2 = cibuild.BuildEnviron(
-        github_event_name="pull_request",
-        github_ref="refs/tags/qux"
-    )
-    assert github2.is_pull_request
-    assert github2.branch == ""
-    assert github2.tag == "qux"
 
 
 def test_buildenviron_commit():
     # Simulates an ordinary commit on the master branch.
     be = cibuild.BuildEnviron(
-        github_ref="refs/heads/master",
+        system="Linux",
-        github_event_name="push",
+        root_dir=root,
+        branch="master",
+        is_pull_request=False,
         should_build_wheel=True,
         should_build_pyinstaller=True,
         should_build_docker=True,
@@ -92,8 +79,8 @@ def test_buildenviron_releasetag():
     # Simulates a tagged release on a release branch.
     be = cibuild.BuildEnviron(
         system="Linux",
-        root_dir="/foo",
+        root_dir=root,
-        github_ref="refs/tags/v0.0.1",
+        tag="v0.0.1",
         should_build_wheel=True,
         should_build_docker=True,
         should_build_pyinstaller=True,
@@ -102,7 +89,7 @@ def test_buildenviron_releasetag():
         docker_password="bar",
     )
     assert be.tag == "v0.0.1"
-    assert be.branch == ""
+    assert be.branch is None
     assert be.version == "0.0.1"
     assert be.upload_dir == "0.0.1"
     assert be.docker_tag == "mitmproxy/mitmproxy:0.0.1"
@@ -116,8 +103,8 @@ def test_buildenviron_namedtag():
     # Simulates a non-release tag on a branch.
     be = cibuild.BuildEnviron(
         system="Linux",
-        root_dir="/foo",
+        root_dir=root,
-        github_ref="refs/tags/anyname",
+        tag="anyname",
         should_build_wheel=True,
         should_build_docker=True,
         should_build_pyinstaller=True,
@@ -126,7 +113,7 @@ def test_buildenviron_namedtag():
         docker_password="bar",
     )
     assert be.tag == "anyname"
-    assert be.branch == ""
+    assert be.branch is None
     assert be.version == "anyname"
     assert be.upload_dir == "anyname"
     assert be.docker_tag == "mitmproxy/mitmproxy:anyname"
@@ -140,8 +127,8 @@ def test_buildenviron_dev_branch():
     # Simulates a commit on a development branch on the main repo
     be = cibuild.BuildEnviron(
         system="Linux",
-        root_dir="/foo",
+        root_dir=root,
-        github_ref="refs/heads/mybranch",
+        branch="mybranch",
         should_build_wheel=True,
         should_build_docker=True,
         should_build_pyinstaller=True,
@@ -149,7 +136,7 @@ def test_buildenviron_dev_branch():
         docker_username="foo",
         docker_password="bar",
     )
-    assert be.tag == ""
+    assert be.tag is None
     assert be.branch == "mybranch"
     assert be.version == "mybranch"
     assert be.upload_dir == "branches/mybranch"
@@ -162,8 +149,8 @@ def test_buildenviron_maintenance_branch():
     # Simulates a commit on a release maintenance branch on the main repo
     be = cibuild.BuildEnviron(
         system="Linux",
-        root_dir="/foo",
+        root_dir=root,
-        github_ref="refs/heads/v0.x",
+        branch="v0.x",
         should_build_wheel=True,
         should_build_docker=True,
         should_build_pyinstaller=True,
@@ -171,7 +158,7 @@ def test_buildenviron_maintenance_branch():
         docker_username="foo",
         docker_password="bar",
     )
-    assert be.tag == ""
+    assert be.tag is None
     assert be.branch == "v0.x"
     assert be.version == "v0.x"
     assert be.upload_dir == "branches/v0.x"
@@ -180,38 +167,32 @@ def test_buildenviron_maintenance_branch():
     assert be.is_maintenance_branch
 
 
-def test_buildenviron_osx(tmpdir):
+def test_buildenviron_osx(tmp_path):
     be = cibuild.BuildEnviron(
         system="Darwin",
-        root_dir="/foo",
+        root_dir=root,
-        github_ref="refs/tags/v0.0.1",
+        tag="v0.0.1",
     )
     assert be.platform_tag == "osx"
-    assert be.bdists == {
+    assert be.archive_path == be.dist_dir / "mitmproxy-0.0.1-osx.tar.gz"
-        "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
-    }
-    assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-osx.tar.gz"
 
-    a = be.archive(os.path.join(tmpdir, "arch"))
+    with be.archive(tmp_path / "arch"):
-    assert a
+        pass
-    a.close()
+    assert (tmp_path / "arch").exists()
 
 
-def test_buildenviron_windows(tmpdir):
+def test_buildenviron_windows(tmp_path):
     be = cibuild.BuildEnviron(
         system="Windows",
-        root_dir="/foo",
+        root_dir=root,
-        github_ref="refs/tags/v0.0.1",
+        tag="v0.0.1",
     )
     assert be.platform_tag == "windows"
-    assert be.bdists == {
+    assert be.archive_path == be.dist_dir / "mitmproxy-0.0.1-windows.zip"
-        "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
-    }
-    assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-windows.zip"
 
-    a = be.archive(os.path.join(tmpdir, "arch"))
+    with be.archive(tmp_path / "arch"):
-    assert a
+        pass
-    a.close()
+    assert (tmp_path / "arch").exists()
 
 
 @pytest.mark.parametrize("version, tag, ok", [
@@ -229,7 +210,8 @@ def test_buildenviron_check_version(version, tag, ok, tmpdir):
 
     be = cibuild.BuildEnviron(
         root_dir=tmpdir,
-        github_ref=f"refs/tags/{tag}",
+        system="Windows",
+        tag=tag,
     )
     if ok:
         be.check_version()