Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-22 07:08:10 +00:00
Code Issues Projects Releases Wiki Activity

Windows: build pyinstaller onedir and use it for installer

Browse Source 
This greatly improves startup time as pyinstaller doesn't have to
unpack everything on startup. The same also applies to macOS and Linux,
but there we 1) don't have installers to hide all the files and
2) have a filesystem that deals much better with lots of small files.

Additionally, simplify cibuild to be a bit more reasonable.
This commit is contained in:
Maximilian Hils 2021-01-05 15:56:33 +01:00
parent ca45548289
commit cca242a581
6 changed files with 414 additions and 401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

309
release/cibuild.py
View File

@ -1,9 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import contextlib import contextlib
import glob
import os import os
import pathlib
import platform import platform
import re import re
import shutil import shutil
@ -12,15 +10,17 @@ import sys
import tarfile import tarfile
import urllib.request import urllib.request
import zipfile import zipfile
from dataclasses import dataclass
from pathlib import Path
from typing import Optional, Union
import click import click
import parver
import cryptography.fernet import cryptography.fernet
import parver
@contextlib.contextmanager @contextlib.contextmanager
def chdir(path: str): # pragma: no cover def chdir(path: Path): # pragma: no cover
old_dir = os.getcwd() old_dir = os.getcwd()
os.chdir(path) os.chdir(path)
yield yield
@ -39,6 +39,16 @@ def bool_from_env(envvar: str) -> bool:
return True return True
class ZipFile2(zipfile.ZipFile):
# ZipFile and tarfile have slightly different APIs. Let's fix that.
def add(self, name: str, arcname: str) -> None:
return self.write(name, arcname)
def __enter__(self) -> "ZipFile2":
return self
@dataclass(frozen=True, repr=False)
class BuildEnviron: class BuildEnviron:
PLATFORM_TAGS = { PLATFORM_TAGS = {
"Darwin": "osx", "Darwin": "osx",
@ -46,100 +56,89 @@ class BuildEnviron:
"Linux": "linux", "Linux": "linux",
} }
def __init__( system: str
self, root_dir: Path
*, branch: Optional[str] = None
system="", tag: Optional[str] = None
root_dir="", is_pull_request: bool = True
github_ref="", should_build_wheel: bool = False
github_event_name="", should_build_docker: bool = False
should_build_wheel=False, should_build_pyinstaller: bool = False
should_build_docker=False, should_build_wininstaller: bool = False
should_build_pyinstaller=False, has_aws_creds: bool = False
should_build_wininstaller=False, has_twine_creds: bool = False
has_aws_creds=False, docker_username: Optional[str] = None
has_twine_creds=False, docker_password: Optional[str] = None
docker_username="", build_key: Optional[str] = None
docker_password="",
build_key="",
):
self.system = system
self.root_dir = root_dir
self.should_build_wheel = should_build_wheel
self.should_build_docker = should_build_docker
self.should_build_pyinstaller = should_build_pyinstaller
self.should_build_wininstaller = should_build_wininstaller
self.github_ref = github_ref
self.github_event_name = github_event_name
self.has_aws_creds = has_aws_creds
self.has_twine_creds = has_twine_creds
self.docker_username = docker_username
self.docker_password = docker_password
self.build_key = build_key
@classmethod @classmethod
def from_env(cls): def from_env(cls) -> "BuildEnviron":
branch = None
tag = None
if ref := os.environ.get("GITHUB_REF", ""):
if ref.startswith("refs/heads/"):
branch = ref.replace("refs/heads/", "")
if ref.startswith("refs/pull/"):
branch = "pr-" + ref.split("/")[2]
if ref.startswith("refs/tags/"):
tag = ref.replace("refs/tags/", "")
is_pull_request = os.environ.get("GITHUB_EVENT_NAME", "pull_request") == "pull_request"
is_untrusted = (
is_pull_request
or
os.environ.get("GITHUB_REPOSITORY", "other") != "mitmproxy/mitmproxy"
)
has_creds = any(
x in os.environ for x in
["AWS_ACCESS_KEY_ID",
"TWINE_USERNAME", "TWINE_PASSWORD",
"DOCKER_USERNAME", "DOCKER_PASSWORD",
"CI_BUILD_KEY"]
)
if is_untrusted and has_creds:
raise RuntimeError("Found upload credentials even though we aren't running on CI!")
return cls( return cls(
system=platform.system(), system=platform.system(),
root_dir=os.path.normpath(os.path.join(os.path.dirname(__file__), "..")), root_dir=Path(__file__).parent.parent,
github_ref=os.environ.get("GITHUB_REF", ""), branch=branch,
github_event_name=os.environ.get("GITHUB_EVENT_NAME", ""), tag=tag,
is_pull_request=is_pull_request,
should_build_wheel=bool_from_env("CI_BUILD_WHEEL"), should_build_wheel=bool_from_env("CI_BUILD_WHEEL"),
should_build_pyinstaller=bool_from_env("CI_BUILD_PYINSTALLER"), should_build_pyinstaller=bool_from_env("CI_BUILD_PYINSTALLER"),
should_build_wininstaller=bool_from_env("CI_BUILD_WININSTALLER"), should_build_wininstaller=bool_from_env("CI_BUILD_WININSTALLER"),
should_build_docker=bool_from_env("CI_BUILD_DOCKER"), should_build_docker=bool_from_env("CI_BUILD_DOCKER"),
has_aws_creds=bool_from_env("AWS_ACCESS_KEY_ID"), has_aws_creds=bool_from_env("AWS_ACCESS_KEY_ID"),
has_twine_creds=bool_from_env("TWINE_USERNAME") and bool_from_env("TWINE_PASSWORD"), has_twine_creds=bool_from_env("TWINE_USERNAME") and bool_from_env("TWINE_PASSWORD"),
docker_username=os.environ.get("DOCKER_USERNAME", ""), docker_username=os.environ.get("DOCKER_USERNAME", None),
docker_password=os.environ.get("DOCKER_PASSWORD", ""), docker_password=os.environ.get("DOCKER_PASSWORD", None),
build_key=os.environ.get("CI_BUILD_KEY", ""), build_key=os.environ.get("CI_BUILD_KEY", None),
) )
def archive(self, path): def archive(self, path: Path) -> Union[tarfile.TarFile, ZipFile2]:
# ZipFile and tarfile have slightly different APIs. Fix that.
if self.system == "Windows": if self.system == "Windows":
a = zipfile.ZipFile(path, "w") return ZipFile2(path, "w")
a.add = a.write
return a
else: else:
return tarfile.open(path, "w:gz") return tarfile.open(path, "w:gz")
def archive_name(self, bdist: str) -> str: @property
def archive_path(self) -> Path:
if self.system == "Windows": if self.system == "Windows":
ext = "zip" ext = "zip"
else: else:
ext = "tar.gz" ext = "tar.gz"
return "{project}-{version}-{platform}.{ext}".format( return self.dist_dir / f"mitmproxy-{self.version}-{self.platform_tag}.{ext}"
project=bdist,
version=self.version,
platform=self.platform_tag,
ext=ext
)
@property @property
def bdists(self): def build_dir(self) -> Path:
return { return self.release_dir / "build"
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
}
@property @property
def branch(self) -> str: def dist_dir(self) -> Path:
if self.github_ref and self.github_ref.startswith("refs/heads/"): return self.release_dir / "dist"
return self.github_ref.replace("refs/heads/", "")
if self.github_ref and self.github_ref.startswith("refs/pull/"):
return "pr-" + self.github_ref.split("/")[2]
return ""
@property
def build_dir(self) -> str:
return os.path.join(self.release_dir, "build")
@property
def dist_dir(self) -> str:
return os.path.join(self.release_dir, "dist")
@property @property
def docker_tag(self) -> str: def docker_tag(self) -> str:
@ -159,7 +158,6 @@ class BuildEnviron:
"release_dir", "release_dir",
"build_dir", "build_dir",
"dist_dir", "dist_dir",
"bdists",
"upload_dir", "upload_dir",
"should_build_wheel", "should_build_wheel",
"should_build_pyinstaller", "should_build_pyinstaller",
@ -177,8 +175,7 @@ class BuildEnviron:
Check that version numbers match our conventions. Check that version numbers match our conventions.
Raises a ValueError if there is a mismatch. Raises a ValueError if there is a mismatch.
""" """
with open(pathlib.Path(self.root_dir) / "mitmproxy" / "version.py") as f: contents = (self.root_dir / "mitmproxy" / "version.py").read_text("utf8")
contents = f.read()
match = re.search(r'^VERSION = "(.+?)"', contents, re.M) match = re.search(r'^VERSION = "(.+?)"', contents, re.M)
assert match assert match
version = match.group(1) version = match.group(1)
@ -211,7 +208,7 @@ class BuildEnviron:
@property @property
def is_prod_release(self) -> bool: def is_prod_release(self) -> bool:
if not (self.tag and self.tag.startswith("v")): if not self.tag or not self.tag.startswith("v"):
return False return False
try: try:
v = parver.Version.parse(self.version, strict=True) v = parver.Version.parse(self.version, strict=True)
@ -219,21 +216,15 @@ class BuildEnviron:
return False return False
return not v.is_prerelease return not v.is_prerelease
@property
def is_pull_request(self) -> bool:
if self.github_event_name == "pull_request":
return True
return False
@property @property
def platform_tag(self) -> str: def platform_tag(self) -> str:
if self.system in self.PLATFORM_TAGS: if self.system in self.PLATFORM_TAGS:
return self.PLATFORM_TAGS[self.system] return self.PLATFORM_TAGS[self.system]
raise BuildError("Unsupported platform: %s" % self.system) raise BuildError(f"Unsupported platform: {self.system}")
@property @property
def release_dir(self) -> str: def release_dir(self) -> Path:
return os.path.join(self.root_dir, "release") return self.root_dir / "release"
@property @property
def should_upload_docker(self) -> bool: def should_upload_docker(self) -> bool:
@ -258,18 +249,12 @@ class BuildEnviron:
self.has_twine_creds, self.has_twine_creds,
]) ])
@property
def tag(self) -> str:
if self.github_ref and self.github_ref.startswith("refs/tags/"):
return self.github_ref.replace("refs/tags/", "")
return ""
@property @property
def upload_dir(self) -> str: def upload_dir(self) -> str:
if self.tag: if self.tag:
return self.version return self.version
else: else:
return "branches/%s" % self.version return f"branches/{self.version}"
@property @property
def version(self) -> str: def version(self) -> str:
@ -287,7 +272,7 @@ class BuildEnviron:
raise BuildError("We're on neither a tag nor a branch - could not establish version") raise BuildError("We're on neither a tag nor a branch - could not establish version")
def build_wheel(be: BuildEnviron): # pragma: no cover def build_wheel(be: BuildEnviron) -> None: # pragma: no cover
click.echo("Building wheel...") click.echo("Building wheel...")
subprocess.check_call([ subprocess.check_call([
"python", "python",
@ -296,22 +281,21 @@ def build_wheel(be: BuildEnviron): # pragma: no cover
"bdist_wheel", "bdist_wheel",
"--dist-dir", be.dist_dir, "--dist-dir", be.dist_dir,
]) ])
whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl')) whl, = be.dist_dir.glob('mitmproxy-*-py3-none-any.whl')
click.echo(f"Found wheel package: {whl}") click.echo(f"Found wheel package: {whl}")
subprocess.check_call(["tox", "-e", "wheeltest", "--", whl]) subprocess.check_call(["tox", "-e", "wheeltest", "--", whl])
return whl
def build_docker_image(be: BuildEnviron): # pragma: no cover def build_docker_image(be: BuildEnviron) -> None: # pragma: no cover
whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl')) whl, = be.dist_dir.glob('mitmproxy-*-py3-none-any.whl')
whl = str(pathlib.Path(whl).relative_to(pathlib.Path(".").absolute())) whl = whl.relative_to(Path(".").absolute())
click.echo("Building Docker images...") click.echo("Building Docker images...")
subprocess.check_call([ subprocess.check_call([
"docker", "docker",
"build", "build",
"--tag", be.docker_tag, "--tag", be.docker_tag,
"--build-arg", f"WHEEL_MITMPROXY={whl}", "--build-arg", f"WHEEL_MITMPROXY={whl}",
"--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(os.path.basename(whl)), "--build-arg", f"WHEEL_BASENAME_MITMPROXY={whl.name}",
"--file", "release/docker/Dockerfile", "--file", "release/docker/Dockerfile",
"." "."
]) ])
@ -328,40 +312,44 @@ def build_docker_image(be: BuildEnviron): # pragma: no cover
assert "Mitmproxy: " in r.stdout.decode() assert "Mitmproxy: " in r.stdout.decode()
def build_pyinstaller(be: BuildEnviron): # pragma: no cover def build_pyinstaller(be: BuildEnviron) -> None: # pragma: no cover
click.echo("Building pyinstaller package...") click.echo("Building pyinstaller package...")
PYINSTALLER_SPEC = os.path.join(be.release_dir, "specs") PYINSTALLER_SPEC = be.release_dir / "specs"
# PyInstaller 3.2 does not bundle pydivert's Windivert binaries PYINSTALLER_HOOKS = be.release_dir / "hooks"
PYINSTALLER_HOOKS = os.path.abspath(os.path.join(be.release_dir, "hooks")) PYINSTALLER_TEMP = be.build_dir / "pyinstaller"
PYINSTALLER_TEMP = os.path.abspath(os.path.join(be.build_dir, "pyinstaller")) PYINSTALLER_DIST = be.build_dir / "binaries" / be.platform_tag
PYINSTALLER_DIST = os.path.abspath(os.path.join(be.build_dir, "binaries", be.platform_tag))
# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes if PYINSTALLER_TEMP.exists():
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
if platform.system() == "Windows":
PYINSTALLER_ARGS = [
# PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
"-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
]
else:
PYINSTALLER_ARGS = []
if os.path.exists(PYINSTALLER_TEMP):
shutil.rmtree(PYINSTALLER_TEMP) shutil.rmtree(PYINSTALLER_TEMP)
if os.path.exists(PYINSTALLER_DIST): if PYINSTALLER_DIST.exists():
shutil.rmtree(PYINSTALLER_DIST) shutil.rmtree(PYINSTALLER_DIST)
for bdist, tools in sorted(be.bdists.items()): if be.platform_tag == "windows":
with be.archive(os.path.join(be.dist_dir, be.archive_name(bdist))) as archive: with chdir(PYINSTALLER_SPEC):
for tool in tools: click.echo("Building PyInstaller binaries in directory mode...")
subprocess.check_call(
[
"pyinstaller",
"--clean",
"--workpath", PYINSTALLER_TEMP,
"--distpath", PYINSTALLER_DIST,
"./windows-dir.spec"
]
)
for tool in ["mitmproxy", "mitmdump", "mitmweb"]:
click.echo(f"> {tool} --version")
executable = (PYINSTALLER_DIST / "onedir" / tool).with_suffix(".exe")
click.echo(subprocess.check_output([executable, "--version"]).decode())
with be.archive(be.archive_path) as archive:
for tool in ["mitmproxy", "mitmdump", "mitmweb"]:
# We can't have a folder and a file with the same name. # We can't have a folder and a file with the same name.
if tool == "mitmproxy": if tool == "mitmproxy":
tool = "mitmproxy_main" tool = "mitmproxy_main"
# This is PyInstaller, so it messes up paths. # Make sure that we are in the spec folder.
# We need to make sure that we are in the spec folder.
with chdir(PYINSTALLER_SPEC): with chdir(PYINSTALLER_SPEC):
click.echo("Building PyInstaller %s binary..." % tool) click.echo(f"Building PyInstaller {tool} binary...")
excludes = [] excludes = []
if tool != "mitmweb": if tool != "mitmweb":
excludes.append("mitmproxy.tools.web") excludes.append("mitmproxy.tools.web")
@ -369,7 +357,7 @@ def build_pyinstaller(be: BuildEnviron): # pragma: no cover
excludes.append("mitmproxy.tools.console") excludes.append("mitmproxy.tools.console")
subprocess.check_call( subprocess.check_call(
[ [ # type: ignore
"pyinstaller", "pyinstaller",
"--clean", "--clean",
"--workpath", PYINSTALLER_TEMP, "--workpath", PYINSTALLER_TEMP,
@ -378,51 +366,46 @@ def build_pyinstaller(be: BuildEnviron): # pragma: no cover
"--onefile", "--onefile",
"--console", "--console",
"--icon", "icon.ico", "--icon", "icon.ico",
# This is PyInstaller, so setting a
# different log level obviously breaks it :-)
# "--log-level", "WARN",
] ]
+ [x for e in excludes for x in ["--exclude-module", e]] + [x for e in excludes for x in ["--exclude-module", e]]
+ PYINSTALLER_ARGS
+ [tool] + [tool]
) )
# Delete the spec file - we're good without. # Delete the spec file - we're good without.
os.remove(f"{tool}.spec") os.remove(f"{tool}.spec")
# Test if it works at all O:-) executable = PYINSTALLER_DIST / tool
executable = os.path.join(PYINSTALLER_DIST, tool) if be.platform_tag == "windows":
if platform.system() == "Windows": executable = executable.with_suffix(".exe")
executable += ".exe"
# Remove _main suffix from mitmproxy executable # Remove _main suffix from mitmproxy executable
if "_main" in executable: if "_main" in executable.name:
shutil.move( executable = executable.rename(
executable, executable.with_name(executable.name.replace("_main", ""))
executable.replace("_main", "")
) )
executable = executable.replace("_main", "")
click.echo("> %s --version" % executable) # Test if it works at all O:-)
click.echo(f"> {executable} --version")
click.echo(subprocess.check_output([executable, "--version"]).decode()) click.echo(subprocess.check_output([executable, "--version"]).decode())
archive.add(executable, os.path.basename(executable)) archive.add(str(executable), str(executable.name))
click.echo("Packed {}.".format(be.archive_name(bdist))) click.echo("Packed {}.".format(be.archive_path.name))
def build_wininstaller(be: BuildEnviron): # pragma: no cover def build_wininstaller(be: BuildEnviron) -> None: # pragma: no cover
if not be.build_key:
click.echo("Cannot build windows installer without secret key.")
return
click.echo("Building wininstaller package...") click.echo("Building wininstaller package...")
IB_VERSION = "20.9.0" IB_VERSION = "20.12.0"
IB_DIR = pathlib.Path(be.release_dir) / "installbuilder" IB_DIR = be.release_dir / "installbuilder"
IB_SETUP = IB_DIR / "setup" / f"{IB_VERSION}-installer.exe" IB_SETUP = IB_DIR / "setup" / f"{IB_VERSION}-installer.exe"
IB_CLI = fr"C:\Program Files (x86)\VMware InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe" IB_CLI = Path(fr"C:\Program Files\VMware InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe")
IB_LICENSE = IB_DIR / "license.xml" IB_LICENSE = IB_DIR / "license.xml"
if not os.path.isfile(IB_CLI): if not IB_LICENSE.exists() and not be.build_key:
if not os.path.isfile(IB_SETUP): click.echo("Cannot build windows installer without secret key.")
return
if not IB_CLI.exists():
if not IB_SETUP.exists():
click.echo("Downloading InstallBuilder...") click.echo("Downloading InstallBuilder...")
def report(block, blocksize, total): def report(block, blocksize, total):
@ -430,17 +413,20 @@ def build_wininstaller(be: BuildEnviron): # pragma: no cover
if round(100 * done / total) != round(100 * (done - blocksize) / total): if round(100 * done / total) != round(100 * (done - blocksize) / total):
click.secho(f"Downloading... {round(100 * done / total)}%") click.secho(f"Downloading... {round(100 * done / total)}%")
tmp = IB_SETUP.with_suffix(".tmp")
urllib.request.urlretrieve( urllib.request.urlretrieve(
f"https://clients.bitrock.com/installbuilder/installbuilder-enterprise-{IB_VERSION}-windows-installer.exe", f"https://clients.bitrock.com/installbuilder/installbuilder-enterprise-{IB_VERSION}-windows-x64-installer.exe",
IB_SETUP.with_suffix(".tmp"), tmp,
reporthook=report reporthook=report
) )
shutil.move(str(IB_SETUP.with_suffix(".tmp")), str(IB_SETUP)) tmp.rename(IB_SETUP)
click.echo("Install InstallBuilder...") click.echo("Install InstallBuilder...")
subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"], check=True) subprocess.run([IB_SETUP, "--mode", "unattended", "--unattendedmodeui", "none"], check=True)
assert os.path.isfile(IB_CLI) assert IB_CLI.is_file()
if not IB_LICENSE.exists():
assert be.build_key
click.echo("Decrypt InstallBuilder license...") click.echo("Decrypt InstallBuilder license...")
f = cryptography.fernet.Fernet(be.build_key.encode()) f = cryptography.fernet.Fernet(be.build_key.encode())
with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, \ with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, \
@ -457,8 +443,7 @@ def build_wininstaller(be: BuildEnviron): # pragma: no cover
"--setvars", f"project.version={be.version}", "--setvars", f"project.version={be.version}",
"--verbose" "--verbose"
], check=True) ], check=True)
assert os.path.isfile( assert (be.dist_dir / f"mitmproxy-{be.version}-windows-installer.exe").exists()
os.path.join(be.dist_dir, f"mitmproxy-{be.version}-windows-installer.exe"))
@click.group(chain=True) @click.group(chain=True)
@ -507,18 +492,18 @@ def upload(): # pragma: no cover
return return
if be.should_upload_aws: if be.should_upload_aws:
num_files = len([name for name in os.listdir(be.dist_dir) if os.path.isfile(name)]) num_files = len([name for name in be.dist_dir.iterdir() if name.is_file()])
click.echo(f"Uploading {num_files} files to AWS dir {be.upload_dir}...") click.echo(f"Uploading {num_files} files to AWS dir {be.upload_dir}...")
subprocess.check_call([ subprocess.check_call([
"aws", "s3", "cp", "aws", "s3", "cp",
"--acl", "public-read", "--acl", "public-read",
be.dist_dir + "/", f"{be.dist_dir}/",
f"s3://snapshots.mitmproxy.org/{be.upload_dir}/", f"s3://snapshots.mitmproxy.org/{be.upload_dir}/",
"--recursive", "--recursive",
]) ])
if be.should_upload_pypi: if be.should_upload_pypi:
whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0] whl, = be.dist_dir.glob('mitmproxy-*-py3-none-any.whl')
click.echo(f"Uploading {whl} to PyPi...") click.echo(f"Uploading {whl} to PyPi...")
subprocess.check_call(["twine", "upload", whl]) subprocess.check_call(["twine", "upload", whl])

2
release/installbuilder/mitmproxy.xml
View File

@ -31,7 +31,7 @@
<distributionFileList> <distributionFileList>
<distributionFile> <distributionFile>
<allowWildcards>1</allowWildcards> <allowWildcards>1</allowWildcards>
<origin>../build/binaries/${platform_name}/*</origin> <origin>../build/binaries/windows/onedir/*</origin>
</distributionFile> </distributionFile>
<distributionFile> <distributionFile>
<origin>run.ps1</origin> <origin>run.ps1</origin>

5
release/installbuilder/run.ps1
View File

@ -1,5 +1,6 @@
$tool = $args[0]
if (Get-Command wt -ErrorAction SilentlyContinue) { if (Get-Command wt -ErrorAction SilentlyContinue) {
Start-Process wt -ArgumentList "powershell.exe","-NoExit","-Command",$args[0] Start-Process wt -ArgumentList "powershell.exe","-Command","& '$PSScriptRoot\$tool.exe'"
} else { } else {
Start-Process powershell -ArgumentList "-NoExit","-Command",$args[0] Start-Process powershell -ArgumentList "-Command","& '$PSScriptRoot\$tool.exe'"
} }

3
release/specs/mitmproxy Normal file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env python
from mitmproxy.tools.main import mitmproxy
mitmproxy()

42
release/specs/windows-dir.spec Normal file
View File

@ -0,0 +1,42 @@
from pathlib import Path
from PyInstaller.building.api import PYZ, EXE, COLLECT
from PyInstaller.building.build_main import Analysis
assert SPECPATH == "."
here = Path(r".")
tools = ["mitmproxy", "mitmdump", "mitmweb"]
analysis = Analysis(
tools,
excludes=["tcl", "tk", "tkinter"],
pathex=[str(here)],
hookspath=[str(here / ".." / "hooks")],
)
pyz = PYZ(analysis.pure, analysis.zipped_data)
executables = []
for tool in tools:
executables.append(EXE(
pyz,
# analysis.scripts has all runtime hooks and all of our tools.
# remove the other tools.
[s for s in analysis.scripts if s[0] not in tools or s[0] == tool],
[],
exclude_binaries=True,
name=tool,
console=True,
upx=False,
icon='icon.ico'
))
COLLECT(
*executables,
analysis.binaries,
analysis.zipfiles,
analysis.datas,
strip=False,
upx=False,
name="onedir"
)

118
test/release/test_cibuild.py
View File

@ -1,10 +1,12 @@
import io import io
import os from pathlib import Path
import pytest import pytest
from release import cibuild from release import cibuild
root = Path(__file__).parent.parent.parent
def test_buildenviron_live(): def test_buildenviron_live():
be = cibuild.BuildEnviron.from_env() be = cibuild.BuildEnviron.from_env()
@ -14,13 +16,12 @@ def test_buildenviron_live():
def test_buildenviron_common(): def test_buildenviron_common():
be = cibuild.BuildEnviron( be = cibuild.BuildEnviron(
system="Linux", system="Linux",
root_dir="/foo", root_dir=root,
github_ref="refs/heads/master", branch="master",
) )
assert be.release_dir == os.path.join(be.root_dir, "release") assert be.release_dir == be.root_dir / "release"
assert be.dist_dir == os.path.join(be.root_dir, "release", "dist") assert be.dist_dir == be.root_dir / "release" / "dist"
assert be.build_dir == os.path.join(be.root_dir, "release", "build") assert be.build_dir == be.root_dir / "release" / "build"
assert be.is_pull_request is False
assert not be.has_docker_creds assert not be.has_docker_creds
cs = io.StringIO() cs = io.StringIO()
@ -29,7 +30,7 @@ def test_buildenviron_common():
be = cibuild.BuildEnviron( be = cibuild.BuildEnviron(
system="Unknown", system="Unknown",
root_dir="/foo", root_dir=root,
) )
with pytest.raises(cibuild.BuildError): with pytest.raises(cibuild.BuildError):
be.version be.version
@ -37,41 +38,27 @@ def test_buildenviron_common():
be.platform_tag be.platform_tag
def test_buildenviron_pr(): def test_buildenviron_pr(monkeypatch):
# Simulates a PR. We build everything, but don't have access to secret # Simulates a PR. We build everything, but don't have access to secret
# credential env variables. # credential env variables.
be = cibuild.BuildEnviron( monkeypatch.setenv("GITHUB_REF", "refs/pull/42/merge")
github_event_name="pull_request", monkeypatch.setenv("CI_BUILD_WHEEL", "1")
should_build_wheel=True, monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request")
should_build_pyinstaller=True,
should_build_docker=True, be = cibuild.BuildEnviron.from_env()
) assert be.branch == "pr-42"
assert be.is_pull_request assert be.is_pull_request
assert be.should_build_wheel
assert not be.should_upload_pypi
def test_ci_systems():
github = cibuild.BuildEnviron(
github_event_name="pull_request",
github_ref="refs/heads/master"
)
assert github.is_pull_request
assert github.branch == "master"
assert github.tag == ""
github2 = cibuild.BuildEnviron(
github_event_name="pull_request",
github_ref="refs/tags/qux"
)
assert github2.is_pull_request
assert github2.branch == ""
assert github2.tag == "qux"
def test_buildenviron_commit(): def test_buildenviron_commit():
# Simulates an ordinary commit on the master branch. # Simulates an ordinary commit on the master branch.
be = cibuild.BuildEnviron( be = cibuild.BuildEnviron(
github_ref="refs/heads/master", system="Linux",
github_event_name="push", root_dir=root,
branch="master",
is_pull_request=False,
should_build_wheel=True, should_build_wheel=True,
should_build_pyinstaller=True, should_build_pyinstaller=True,
should_build_docker=True, should_build_docker=True,
@ -92,8 +79,8 @@ def test_buildenviron_releasetag():
# Simulates a tagged release on a release branch. # Simulates a tagged release on a release branch.
be = cibuild.BuildEnviron( be = cibuild.BuildEnviron(
system="Linux", system="Linux",
root_dir="/foo", root_dir=root,
github_ref="refs/tags/v0.0.1", tag="v0.0.1",
should_build_wheel=True, should_build_wheel=True,
should_build_docker=True, should_build_docker=True,
should_build_pyinstaller=True, should_build_pyinstaller=True,
@ -102,7 +89,7 @@ def test_buildenviron_releasetag():
docker_password="bar", docker_password="bar",
) )
assert be.tag == "v0.0.1" assert be.tag == "v0.0.1"
assert be.branch == "" assert be.branch is None
assert be.version == "0.0.1" assert be.version == "0.0.1"
assert be.upload_dir == "0.0.1" assert be.upload_dir == "0.0.1"
assert be.docker_tag == "mitmproxy/mitmproxy:0.0.1" assert be.docker_tag == "mitmproxy/mitmproxy:0.0.1"
@ -116,8 +103,8 @@ def test_buildenviron_namedtag():
# Simulates a non-release tag on a branch. # Simulates a non-release tag on a branch.
be = cibuild.BuildEnviron( be = cibuild.BuildEnviron(
system="Linux", system="Linux",
root_dir="/foo", root_dir=root,
github_ref="refs/tags/anyname", tag="anyname",
should_build_wheel=True, should_build_wheel=True,
should_build_docker=True, should_build_docker=True,
should_build_pyinstaller=True, should_build_pyinstaller=True,
@ -126,7 +113,7 @@ def test_buildenviron_namedtag():
docker_password="bar", docker_password="bar",
) )
assert be.tag == "anyname" assert be.tag == "anyname"
assert be.branch == "" assert be.branch is None
assert be.version == "anyname" assert be.version == "anyname"
assert be.upload_dir == "anyname" assert be.upload_dir == "anyname"
assert be.docker_tag == "mitmproxy/mitmproxy:anyname" assert be.docker_tag == "mitmproxy/mitmproxy:anyname"
@ -140,8 +127,8 @@ def test_buildenviron_dev_branch():
# Simulates a commit on a development branch on the main repo # Simulates a commit on a development branch on the main repo
be = cibuild.BuildEnviron( be = cibuild.BuildEnviron(
system="Linux", system="Linux",
root_dir="/foo", root_dir=root,
github_ref="refs/heads/mybranch", branch="mybranch",
should_build_wheel=True, should_build_wheel=True,
should_build_docker=True, should_build_docker=True,
should_build_pyinstaller=True, should_build_pyinstaller=True,
@ -149,7 +136,7 @@ def test_buildenviron_dev_branch():
docker_username="foo", docker_username="foo",
docker_password="bar", docker_password="bar",
) )
assert be.tag == "" assert be.tag is None
assert be.branch == "mybranch" assert be.branch == "mybranch"
assert be.version == "mybranch" assert be.version == "mybranch"
assert be.upload_dir == "branches/mybranch" assert be.upload_dir == "branches/mybranch"
@ -162,8 +149,8 @@ def test_buildenviron_maintenance_branch():
# Simulates a commit on a release maintenance branch on the main repo # Simulates a commit on a release maintenance branch on the main repo
be = cibuild.BuildEnviron( be = cibuild.BuildEnviron(
system="Linux", system="Linux",
root_dir="/foo", root_dir=root,
github_ref="refs/heads/v0.x", branch="v0.x",
should_build_wheel=True, should_build_wheel=True,
should_build_docker=True, should_build_docker=True,
should_build_pyinstaller=True, should_build_pyinstaller=True,
@ -171,7 +158,7 @@ def test_buildenviron_maintenance_branch():
docker_username="foo", docker_username="foo",
docker_password="bar", docker_password="bar",
) )
assert be.tag == "" assert be.tag is None
assert be.branch == "v0.x" assert be.branch == "v0.x"
assert be.version == "v0.x" assert be.version == "v0.x"
assert be.upload_dir == "branches/v0.x" assert be.upload_dir == "branches/v0.x"
@ -180,38 +167,32 @@ def test_buildenviron_maintenance_branch():
assert be.is_maintenance_branch assert be.is_maintenance_branch
def test_buildenviron_osx(tmpdir): def test_buildenviron_osx(tmp_path):
be = cibuild.BuildEnviron( be = cibuild.BuildEnviron(
system="Darwin", system="Darwin",
root_dir="/foo", root_dir=root,
github_ref="refs/tags/v0.0.1", tag="v0.0.1",
) )
assert be.platform_tag == "osx" assert be.platform_tag == "osx"
assert be.bdists == { assert be.archive_path == be.dist_dir / "mitmproxy-0.0.1-osx.tar.gz"
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
}
assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-osx.tar.gz"
a = be.archive(os.path.join(tmpdir, "arch")) with be.archive(tmp_path / "arch"):
assert a pass
a.close() assert (tmp_path / "arch").exists()
def test_buildenviron_windows(tmpdir): def test_buildenviron_windows(tmp_path):
be = cibuild.BuildEnviron( be = cibuild.BuildEnviron(
system="Windows", system="Windows",
root_dir="/foo", root_dir=root,
github_ref="refs/tags/v0.0.1", tag="v0.0.1",
) )
assert be.platform_tag == "windows" assert be.platform_tag == "windows"
assert be.bdists == { assert be.archive_path == be.dist_dir / "mitmproxy-0.0.1-windows.zip"
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
}
assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-windows.zip"
a = be.archive(os.path.join(tmpdir, "arch")) with be.archive(tmp_path / "arch"):
assert a pass
a.close() assert (tmp_path / "arch").exists()
@pytest.mark.parametrize("version, tag, ok", [ @pytest.mark.parametrize("version, tag, ok", [
@ -229,7 +210,8 @@ def test_buildenviron_check_version(version, tag, ok, tmpdir):
be = cibuild.BuildEnviron( be = cibuild.BuildEnviron(
root_dir=tmpdir, root_dir=tmpdir,
github_ref=f"refs/tags/{tag}", system="Windows",
tag=tag,
) )
if ok: if ok:
be.check_version() be.check_version()