mirror of
https://github.com/Grasscutters/mitmproxy.git
synced 2024-11-21 22:58:24 +00:00
Windows: build pyinstaller onedir and use it for installer
This greatly improves startup time as pyinstaller doesn't have to unpack everything on startup. The same also applies to macOS and Linux, but there we 1) don't have installers to hide all the files and 2) have a filesystem that deals much better with lots of small files. Additionally, simplify cibuild to be a bit more reasonable.
This commit is contained in:
Maximilian Hils
parent
ca45548289
commit
cca242a581
@ -1,9 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import contextlib
|
||||
import glob
|
||||
import os
|
||||
import pathlib
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
@ -12,15 +10,17 @@ import sys
|
||||
import tarfile
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
import click
|
||||
import parver
|
||||
|
||||
import cryptography.fernet
|
||||
import parver
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def chdir(path: str): # pragma: no cover
|
||||
def chdir(path: Path): # pragma: no cover
|
||||
old_dir = os.getcwd()
|
||||
os.chdir(path)
|
||||
yield
|
||||
@ -39,6 +39,16 @@ def bool_from_env(envvar: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class ZipFile2(zipfile.ZipFile):
|
||||
# ZipFile and tarfile have slightly different APIs. Let's fix that.
|
||||
def add(self, name: str, arcname: str) -> None:
|
||||
return self.write(name, arcname)
|
||||
|
||||
def __enter__(self) -> "ZipFile2":
|
||||
return self
|
||||
|
||||
|
||||
@dataclass(frozen=True, repr=False)
|
||||
class BuildEnviron:
|
||||
PLATFORM_TAGS = {
|
||||
"Darwin": "osx",
|
||||
@ -46,100 +56,89 @@ class BuildEnviron:
|
||||
"Linux": "linux",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
system="",
|
||||
root_dir="",
|
||||
github_ref="",
|
||||
github_event_name="",
|
||||
should_build_wheel=False,
|
||||
should_build_docker=False,
|
||||
should_build_pyinstaller=False,
|
||||
should_build_wininstaller=False,
|
||||
has_aws_creds=False,
|
||||
has_twine_creds=False,
|
||||
docker_username="",
|
||||
docker_password="",
|
||||
build_key="",
|
||||
):
|
||||
self.system = system
|
||||
self.root_dir = root_dir
|
||||
|
||||
self.should_build_wheel = should_build_wheel
|
||||
self.should_build_docker = should_build_docker
|
||||
self.should_build_pyinstaller = should_build_pyinstaller
|
||||
self.should_build_wininstaller = should_build_wininstaller
|
||||
|
||||
self.github_ref = github_ref
|
||||
self.github_event_name = github_event_name
|
||||
|
||||
self.has_aws_creds = has_aws_creds
|
||||
self.has_twine_creds = has_twine_creds
|
||||
self.docker_username = docker_username
|
||||
self.docker_password = docker_password
|
||||
self.build_key = build_key
|
||||
system: str
|
||||
root_dir: Path
|
||||
branch: Optional[str] = None
|
||||
tag: Optional[str] = None
|
||||
is_pull_request: bool = True
|
||||
should_build_wheel: bool = False
|
||||
should_build_docker: bool = False
|
||||
should_build_pyinstaller: bool = False
|
||||
should_build_wininstaller: bool = False
|
||||
has_aws_creds: bool = False
|
||||
has_twine_creds: bool = False
|
||||
docker_username: Optional[str] = None
|
||||
docker_password: Optional[str] = None
|
||||
build_key: Optional[str] = None
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
def from_env(cls) -> "BuildEnviron":
|
||||
branch = None
|
||||
tag = None
|
||||
|
||||
if ref := os.environ.get("GITHUB_REF", ""):
|
||||
if ref.startswith("refs/heads/"):
|
||||
branch = ref.replace("refs/heads/", "")
|
||||
if ref.startswith("refs/pull/"):
|
||||
branch = "pr-" + ref.split("/")[2]
|
||||
if ref.startswith("refs/tags/"):
|
||||
tag = ref.replace("refs/tags/", "")
|
||||
|
||||
is_pull_request = os.environ.get("GITHUB_EVENT_NAME", "pull_request") == "pull_request"
|
||||
|
||||
is_untrusted = (
|
||||
is_pull_request
|
||||
or
|
||||
os.environ.get("GITHUB_REPOSITORY", "other") != "mitmproxy/mitmproxy"
|
||||
)
|
||||
has_creds = any(
|
||||
x in os.environ for x in
|
||||
["AWS_ACCESS_KEY_ID",
|
||||
"TWINE_USERNAME", "TWINE_PASSWORD",
|
||||
"DOCKER_USERNAME", "DOCKER_PASSWORD",
|
||||
"CI_BUILD_KEY"]
|
||||
)
|
||||
if is_untrusted and has_creds:
|
||||
raise RuntimeError("Found upload credentials even though we aren't running on CI!")
|
||||
|
||||
return cls(
|
||||
system=platform.system(),
|
||||
root_dir=os.path.normpath(os.path.join(os.path.dirname(__file__), "..")),
|
||||
github_ref=os.environ.get("GITHUB_REF", ""),
|
||||
github_event_name=os.environ.get("GITHUB_EVENT_NAME", ""),
|
||||
root_dir=Path(__file__).parent.parent,
|
||||
branch=branch,
|
||||
tag=tag,
|
||||
is_pull_request=is_pull_request,
|
||||
should_build_wheel=bool_from_env("CI_BUILD_WHEEL"),
|
||||
should_build_pyinstaller=bool_from_env("CI_BUILD_PYINSTALLER"),
|
||||
should_build_wininstaller=bool_from_env("CI_BUILD_WININSTALLER"),
|
||||
should_build_docker=bool_from_env("CI_BUILD_DOCKER"),
|
||||
has_aws_creds=bool_from_env("AWS_ACCESS_KEY_ID"),
|
||||
has_twine_creds=bool_from_env("TWINE_USERNAME") and bool_from_env("TWINE_PASSWORD"),
|
||||
docker_username=os.environ.get("DOCKER_USERNAME", ""),
|
||||
docker_password=os.environ.get("DOCKER_PASSWORD", ""),
|
||||
build_key=os.environ.get("CI_BUILD_KEY", ""),
|
||||
docker_username=os.environ.get("DOCKER_USERNAME", None),
|
||||
docker_password=os.environ.get("DOCKER_PASSWORD", None),
|
||||
build_key=os.environ.get("CI_BUILD_KEY", None),
|
||||
)
|
||||
|
||||
def archive(self, path):
|
||||
# ZipFile and tarfile have slightly different APIs. Fix that.
|
||||
def archive(self, path: Path) -> Union[tarfile.TarFile, ZipFile2]:
|
||||
if self.system == "Windows":
|
||||
a = zipfile.ZipFile(path, "w")
|
||||
a.add = a.write
|
||||
return a
|
||||
return ZipFile2(path, "w")
|
||||
else:
|
||||
return tarfile.open(path, "w:gz")
|
||||
|
||||
def archive_name(self, bdist: str) -> str:
|
||||
@property
|
||||
def archive_path(self) -> Path:
|
||||
if self.system == "Windows":
|
||||
ext = "zip"
|
||||
else:
|
||||
ext = "tar.gz"
|
||||
return "{project}-{version}-{platform}.{ext}".format(
|
||||
project=bdist,
|
||||
version=self.version,
|
||||
platform=self.platform_tag,
|
||||
ext=ext
|
||||
)
|
||||
return self.dist_dir / f"mitmproxy-{self.version}-{self.platform_tag}.{ext}"
|
||||
|
||||
@property
|
||||
def bdists(self):
|
||||
return {
|
||||
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
|
||||
}
|
||||
def build_dir(self) -> Path:
|
||||
return self.release_dir / "build"
|
||||
|
||||
@property
|
||||
def branch(self) -> str:
|
||||
if self.github_ref and self.github_ref.startswith("refs/heads/"):
|
||||
return self.github_ref.replace("refs/heads/", "")
|
||||
if self.github_ref and self.github_ref.startswith("refs/pull/"):
|
||||
return "pr-" + self.github_ref.split("/")[2]
|
||||
return ""
|
||||
|
||||
@property
|
||||
def build_dir(self) -> str:
|
||||
return os.path.join(self.release_dir, "build")
|
||||
|
||||
@property
|
||||
def dist_dir(self) -> str:
|
||||
return os.path.join(self.release_dir, "dist")
|
||||
def dist_dir(self) -> Path:
|
||||
return self.release_dir / "dist"
|
||||
|
||||
@property
|
||||
def docker_tag(self) -> str:
|
||||
@ -159,7 +158,6 @@ class BuildEnviron:
|
||||
"release_dir",
|
||||
"build_dir",
|
||||
"dist_dir",
|
||||
"bdists",
|
||||
"upload_dir",
|
||||
"should_build_wheel",
|
||||
"should_build_pyinstaller",
|
||||
@ -177,8 +175,7 @@ class BuildEnviron:
|
||||
Check that version numbers match our conventions.
|
||||
Raises a ValueError if there is a mismatch.
|
||||
"""
|
||||
with open(pathlib.Path(self.root_dir) / "mitmproxy" / "version.py") as f:
|
||||
contents = f.read()
|
||||
contents = (self.root_dir / "mitmproxy" / "version.py").read_text("utf8")
|
||||
match = re.search(r'^VERSION = "(.+?)"', contents, re.M)
|
||||
assert match
|
||||
version = match.group(1)
|
||||
@ -211,7 +208,7 @@ class BuildEnviron:
|
||||
|
||||
@property
|
||||
def is_prod_release(self) -> bool:
|
||||
if not (self.tag and self.tag.startswith("v")):
|
||||
if not self.tag or not self.tag.startswith("v"):
|
||||
return False
|
||||
try:
|
||||
v = parver.Version.parse(self.version, strict=True)
|
||||
@ -219,21 +216,15 @@ class BuildEnviron:
|
||||
return False
|
||||
return not v.is_prerelease
|
||||
|
||||
@property
|
||||
def is_pull_request(self) -> bool:
|
||||
if self.github_event_name == "pull_request":
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def platform_tag(self) -> str:
|
||||
if self.system in self.PLATFORM_TAGS:
|
||||
return self.PLATFORM_TAGS[self.system]
|
||||
raise BuildError("Unsupported platform: %s" % self.system)
|
||||
raise BuildError(f"Unsupported platform: {self.system}")
|
||||
|
||||
@property
|
||||
def release_dir(self) -> str:
|
||||
return os.path.join(self.root_dir, "release")
|
||||
def release_dir(self) -> Path:
|
||||
return self.root_dir / "release"
|
||||
|
||||
@property
|
||||
def should_upload_docker(self) -> bool:
|
||||
@ -258,18 +249,12 @@ class BuildEnviron:
|
||||
self.has_twine_creds,
|
||||
])
|
||||
|
||||
@property
|
||||
def tag(self) -> str:
|
||||
if self.github_ref and self.github_ref.startswith("refs/tags/"):
|
||||
return self.github_ref.replace("refs/tags/", "")
|
||||
return ""
|
||||
|
||||
@property
|
||||
def upload_dir(self) -> str:
|
||||
if self.tag:
|
||||
return self.version
|
||||
else:
|
||||
return "branches/%s" % self.version
|
||||
return f"branches/{self.version}"
|
||||
|
||||
@property
|
||||
def version(self) -> str:
|
||||
@ -287,7 +272,7 @@ class BuildEnviron:
|
||||
raise BuildError("We're on neither a tag nor a branch - could not establish version")
|
||||
|
||||
|
||||
def build_wheel(be: BuildEnviron): # pragma: no cover
|
||||
def build_wheel(be: BuildEnviron) -> None: # pragma: no cover
|
||||
click.echo("Building wheel...")
|
||||
subprocess.check_call([
|
||||
"python",
|
||||
@ -296,22 +281,21 @@ def build_wheel(be: BuildEnviron): # pragma: no cover
|
||||
"bdist_wheel",
|
||||
"--dist-dir", be.dist_dir,
|
||||
])
|
||||
whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))
|
||||
whl, = be.dist_dir.glob('mitmproxy-*-py3-none-any.whl')
|
||||
click.echo(f"Found wheel package: {whl}")
|
||||
subprocess.check_call(["tox", "-e", "wheeltest", "--", whl])
|
||||
return whl
|
||||
|
||||
|
||||
def build_docker_image(be: BuildEnviron): # pragma: no cover
|
||||
whl, = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))
|
||||
whl = str(pathlib.Path(whl).relative_to(pathlib.Path(".").absolute()))
|
||||
def build_docker_image(be: BuildEnviron) -> None: # pragma: no cover
|
||||
whl, = be.dist_dir.glob('mitmproxy-*-py3-none-any.whl')
|
||||
whl = whl.relative_to(Path(".").absolute())
|
||||
click.echo("Building Docker images...")
|
||||
subprocess.check_call([
|
||||
"docker",
|
||||
"build",
|
||||
"--tag", be.docker_tag,
|
||||
"--build-arg", f"WHEEL_MITMPROXY={whl}",
|
||||
"--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(os.path.basename(whl)),
|
||||
"--build-arg", f"WHEEL_BASENAME_MITMPROXY={whl.name}",
|
||||
"--file", "release/docker/Dockerfile",
|
||||
"."
|
||||
])
|
||||
@ -328,101 +312,100 @@ def build_docker_image(be: BuildEnviron): # pragma: no cover
|
||||
assert "Mitmproxy: " in r.stdout.decode()
|
||||
|
||||
|
||||
def build_pyinstaller(be: BuildEnviron): # pragma: no cover
|
||||
def build_pyinstaller(be: BuildEnviron) -> None: # pragma: no cover
|
||||
click.echo("Building pyinstaller package...")
|
||||
|
||||
PYINSTALLER_SPEC = os.path.join(be.release_dir, "specs")
|
||||
# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
|
||||
PYINSTALLER_HOOKS = os.path.abspath(os.path.join(be.release_dir, "hooks"))
|
||||
PYINSTALLER_TEMP = os.path.abspath(os.path.join(be.build_dir, "pyinstaller"))
|
||||
PYINSTALLER_DIST = os.path.abspath(os.path.join(be.build_dir, "binaries", be.platform_tag))
|
||||
PYINSTALLER_SPEC = be.release_dir / "specs"
|
||||
PYINSTALLER_HOOKS = be.release_dir / "hooks"
|
||||
PYINSTALLER_TEMP = be.build_dir / "pyinstaller"
|
||||
PYINSTALLER_DIST = be.build_dir / "binaries" / be.platform_tag
|
||||
|
||||
# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
|
||||
# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
|
||||
if platform.system() == "Windows":
|
||||
PYINSTALLER_ARGS = [
|
||||
# PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
|
||||
"-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
|
||||
]
|
||||
else:
|
||||
PYINSTALLER_ARGS = []
|
||||
|
||||
if os.path.exists(PYINSTALLER_TEMP):
|
||||
if PYINSTALLER_TEMP.exists():
|
||||
shutil.rmtree(PYINSTALLER_TEMP)
|
||||
if os.path.exists(PYINSTALLER_DIST):
|
||||
if PYINSTALLER_DIST.exists():
|
||||
shutil.rmtree(PYINSTALLER_DIST)
|
||||
|
||||
for bdist, tools in sorted(be.bdists.items()):
|
||||
with be.archive(os.path.join(be.dist_dir, be.archive_name(bdist))) as archive:
|
||||
for tool in tools:
|
||||
# We can't have a folder and a file with the same name.
|
||||
if tool == "mitmproxy":
|
||||
tool = "mitmproxy_main"
|
||||
# This is PyInstaller, so it messes up paths.
|
||||
# We need to make sure that we are in the spec folder.
|
||||
with chdir(PYINSTALLER_SPEC):
|
||||
click.echo("Building PyInstaller %s binary..." % tool)
|
||||
excludes = []
|
||||
if tool != "mitmweb":
|
||||
excludes.append("mitmproxy.tools.web")
|
||||
if tool != "mitmproxy_main":
|
||||
excludes.append("mitmproxy.tools.console")
|
||||
|
||||
subprocess.check_call(
|
||||
[
|
||||
"pyinstaller",
|
||||
"--clean",
|
||||
"--workpath", PYINSTALLER_TEMP,
|
||||
"--distpath", PYINSTALLER_DIST,
|
||||
"--additional-hooks-dir", PYINSTALLER_HOOKS,
|
||||
"--onefile",
|
||||
"--console",
|
||||
"--icon", "icon.ico",
|
||||
# This is PyInstaller, so setting a
|
||||
# different log level obviously breaks it :-)
|
||||
# "--log-level", "WARN",
|
||||
]
|
||||
+ [x for e in excludes for x in ["--exclude-module", e]]
|
||||
+ PYINSTALLER_ARGS
|
||||
+ [tool]
|
||||
)
|
||||
# Delete the spec file - we're good without.
|
||||
os.remove(f"{tool}.spec")
|
||||
|
||||
# Test if it works at all O:-)
|
||||
executable = os.path.join(PYINSTALLER_DIST, tool)
|
||||
if platform.system() == "Windows":
|
||||
executable += ".exe"
|
||||
|
||||
# Remove _main suffix from mitmproxy executable
|
||||
if "_main" in executable:
|
||||
shutil.move(
|
||||
executable,
|
||||
executable.replace("_main", "")
|
||||
)
|
||||
executable = executable.replace("_main", "")
|
||||
|
||||
click.echo("> %s --version" % executable)
|
||||
if be.platform_tag == "windows":
|
||||
with chdir(PYINSTALLER_SPEC):
|
||||
click.echo("Building PyInstaller binaries in directory mode...")
|
||||
subprocess.check_call(
|
||||
[
|
||||
"pyinstaller",
|
||||
"--clean",
|
||||
"--workpath", PYINSTALLER_TEMP,
|
||||
"--distpath", PYINSTALLER_DIST,
|
||||
"./windows-dir.spec"
|
||||
]
|
||||
)
|
||||
for tool in ["mitmproxy", "mitmdump", "mitmweb"]:
|
||||
click.echo(f"> {tool} --version")
|
||||
executable = (PYINSTALLER_DIST / "onedir" / tool).with_suffix(".exe")
|
||||
click.echo(subprocess.check_output([executable, "--version"]).decode())
|
||||
|
||||
archive.add(executable, os.path.basename(executable))
|
||||
click.echo("Packed {}.".format(be.archive_name(bdist)))
|
||||
with be.archive(be.archive_path) as archive:
|
||||
for tool in ["mitmproxy", "mitmdump", "mitmweb"]:
|
||||
# We can't have a folder and a file with the same name.
|
||||
if tool == "mitmproxy":
|
||||
tool = "mitmproxy_main"
|
||||
# Make sure that we are in the spec folder.
|
||||
with chdir(PYINSTALLER_SPEC):
|
||||
click.echo(f"Building PyInstaller {tool} binary...")
|
||||
excludes = []
|
||||
if tool != "mitmweb":
|
||||
excludes.append("mitmproxy.tools.web")
|
||||
if tool != "mitmproxy_main":
|
||||
excludes.append("mitmproxy.tools.console")
|
||||
|
||||
subprocess.check_call(
|
||||
[ # type: ignore
|
||||
"pyinstaller",
|
||||
"--clean",
|
||||
"--workpath", PYINSTALLER_TEMP,
|
||||
"--distpath", PYINSTALLER_DIST,
|
||||
"--additional-hooks-dir", PYINSTALLER_HOOKS,
|
||||
"--onefile",
|
||||
"--console",
|
||||
"--icon", "icon.ico",
|
||||
]
|
||||
+ [x for e in excludes for x in ["--exclude-module", e]]
|
||||
+ [tool]
|
||||
)
|
||||
# Delete the spec file - we're good without.
|
||||
os.remove(f"{tool}.spec")
|
||||
|
||||
executable = PYINSTALLER_DIST / tool
|
||||
if be.platform_tag == "windows":
|
||||
executable = executable.with_suffix(".exe")
|
||||
|
||||
# Remove _main suffix from mitmproxy executable
|
||||
if "_main" in executable.name:
|
||||
executable = executable.rename(
|
||||
executable.with_name(executable.name.replace("_main", ""))
|
||||
)
|
||||
|
||||
# Test if it works at all O:-)
|
||||
click.echo(f"> {executable} --version")
|
||||
click.echo(subprocess.check_output([executable, "--version"]).decode())
|
||||
|
||||
archive.add(str(executable), str(executable.name))
|
||||
click.echo("Packed {}.".format(be.archive_path.name))
|
||||
|
||||
|
||||
def build_wininstaller(be: BuildEnviron): # pragma: no cover
|
||||
if not be.build_key:
|
||||
click.echo("Cannot build windows installer without secret key.")
|
||||
return
|
||||
def build_wininstaller(be: BuildEnviron) -> None: # pragma: no cover
|
||||
click.echo("Building wininstaller package...")
|
||||
|
||||
IB_VERSION = "20.9.0"
|
||||
IB_DIR = pathlib.Path(be.release_dir) / "installbuilder"
|
||||
IB_VERSION = "20.12.0"
|
||||
IB_DIR = be.release_dir / "installbuilder"
|
||||
IB_SETUP = IB_DIR / "setup" / f"{IB_VERSION}-installer.exe"
|
||||
IB_CLI = fr"C:\Program Files (x86)\VMware InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe"
|
||||
IB_CLI = Path(fr"C:\Program Files\VMware InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe")
|
||||
IB_LICENSE = IB_DIR / "license.xml"
|
||||
|
||||
if not os.path.isfile(IB_CLI):
|
||||
if not os.path.isfile(IB_SETUP):
|
||||
if not IB_LICENSE.exists() and not be.build_key:
|
||||
click.echo("Cannot build windows installer without secret key.")
|
||||
return
|
||||
|
||||
if not IB_CLI.exists():
|
||||
if not IB_SETUP.exists():
|
||||
click.echo("Downloading InstallBuilder...")
|
||||
|
||||
def report(block, blocksize, total):
|
||||
@ -430,22 +413,25 @@ def build_wininstaller(be: BuildEnviron): # pragma: no cover
|
||||
if round(100 * done / total) != round(100 * (done - blocksize) / total):
|
||||
click.secho(f"Downloading... {round(100 * done / total)}%")
|
||||
|
||||
tmp = IB_SETUP.with_suffix(".tmp")
|
||||
urllib.request.urlretrieve(
|
||||
f"https://clients.bitrock.com/installbuilder/installbuilder-enterprise-{IB_VERSION}-windows-installer.exe",
|
||||
IB_SETUP.with_suffix(".tmp"),
|
||||
f"https://clients.bitrock.com/installbuilder/installbuilder-enterprise-{IB_VERSION}-windows-x64-installer.exe",
|
||||
tmp,
|
||||
reporthook=report
|
||||
)
|
||||
shutil.move(str(IB_SETUP.with_suffix(".tmp")), str(IB_SETUP))
|
||||
tmp.rename(IB_SETUP)
|
||||
|
||||
click.echo("Install InstallBuilder...")
|
||||
subprocess.run([str(IB_SETUP), "--mode", "unattended", "--unattendedmodeui", "none"], check=True)
|
||||
assert os.path.isfile(IB_CLI)
|
||||
subprocess.run([IB_SETUP, "--mode", "unattended", "--unattendedmodeui", "none"], check=True)
|
||||
assert IB_CLI.is_file()
|
||||
|
||||
click.echo("Decrypt InstallBuilder license...")
|
||||
f = cryptography.fernet.Fernet(be.build_key.encode())
|
||||
with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, \
|
||||
open(IB_LICENSE, "wb") as outfile:
|
||||
outfile.write(f.decrypt(infile.read()))
|
||||
if not IB_LICENSE.exists():
|
||||
assert be.build_key
|
||||
click.echo("Decrypt InstallBuilder license...")
|
||||
f = cryptography.fernet.Fernet(be.build_key.encode())
|
||||
with open(IB_LICENSE.with_suffix(".xml.enc"), "rb") as infile, \
|
||||
open(IB_LICENSE, "wb") as outfile:
|
||||
outfile.write(f.decrypt(infile.read()))
|
||||
|
||||
click.echo("Run InstallBuilder...")
|
||||
subprocess.run([
|
||||
@ -457,8 +443,7 @@ def build_wininstaller(be: BuildEnviron): # pragma: no cover
|
||||
"--setvars", f"project.version={be.version}",
|
||||
"--verbose"
|
||||
], check=True)
|
||||
assert os.path.isfile(
|
||||
os.path.join(be.dist_dir, f"mitmproxy-{be.version}-windows-installer.exe"))
|
||||
assert (be.dist_dir / f"mitmproxy-{be.version}-windows-installer.exe").exists()
|
||||
|
||||
|
||||
@click.group(chain=True)
|
||||
@ -507,18 +492,18 @@ def upload(): # pragma: no cover
|
||||
return
|
||||
|
||||
if be.should_upload_aws:
|
||||
num_files = len([name for name in os.listdir(be.dist_dir) if os.path.isfile(name)])
|
||||
num_files = len([name for name in be.dist_dir.iterdir() if name.is_file()])
|
||||
click.echo(f"Uploading {num_files} files to AWS dir {be.upload_dir}...")
|
||||
subprocess.check_call([
|
||||
"aws", "s3", "cp",
|
||||
"--acl", "public-read",
|
||||
be.dist_dir + "/",
|
||||
f"{be.dist_dir}/",
|
||||
f"s3://snapshots.mitmproxy.org/{be.upload_dir}/",
|
||||
"--recursive",
|
||||
])
|
||||
|
||||
if be.should_upload_pypi:
|
||||
whl = glob.glob(os.path.join(be.dist_dir, 'mitmproxy-*-py3-none-any.whl'))[0]
|
||||
whl, = be.dist_dir.glob('mitmproxy-*-py3-none-any.whl')
|
||||
click.echo(f"Uploading {whl} to PyPi...")
|
||||
subprocess.check_call(["twine", "upload", whl])
|
||||
|
||||
|
||||
@ -31,7 +31,7 @@
|
||||
<distributionFileList>
|
||||
<distributionFile>
|
||||
<allowWildcards>1</allowWildcards>
|
||||
<origin>../build/binaries/${platform_name}/*</origin>
|
||||
<origin>../build/binaries/windows/onedir/*</origin>
|
||||
</distributionFile>
|
||||
<distributionFile>
|
||||
<origin>run.ps1</origin>
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
$tool = $args[0]
|
||||
if (Get-Command wt -ErrorAction SilentlyContinue) {
|
||||
Start-Process wt -ArgumentList "powershell.exe","-NoExit","-Command",$args[0]
|
||||
Start-Process wt -ArgumentList "powershell.exe","-Command","& '$PSScriptRoot\$tool.exe'"
|
||||
} else {
|
||||
Start-Process powershell -ArgumentList "-NoExit","-Command",$args[0]
|
||||
Start-Process powershell -ArgumentList "-Command","& '$PSScriptRoot\$tool.exe'"
|
||||
}
|
||||
|
||||
3
release/specs/mitmproxy
Normal file
3
release/specs/mitmproxy
Normal file
@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env python
|
||||
from mitmproxy.tools.main import mitmproxy
|
||||
mitmproxy()
|
||||
42
release/specs/windows-dir.spec
Normal file
42
release/specs/windows-dir.spec
Normal file
@ -0,0 +1,42 @@
|
||||
from pathlib import Path
|
||||
|
||||
from PyInstaller.building.api import PYZ, EXE, COLLECT
|
||||
from PyInstaller.building.build_main import Analysis
|
||||
|
||||
assert SPECPATH == "."
|
||||
|
||||
here = Path(r".")
|
||||
tools = ["mitmproxy", "mitmdump", "mitmweb"]
|
||||
|
||||
analysis = Analysis(
|
||||
tools,
|
||||
excludes=["tcl", "tk", "tkinter"],
|
||||
pathex=[str(here)],
|
||||
hookspath=[str(here / ".." / "hooks")],
|
||||
)
|
||||
|
||||
pyz = PYZ(analysis.pure, analysis.zipped_data)
|
||||
executables = []
|
||||
for tool in tools:
|
||||
executables.append(EXE(
|
||||
pyz,
|
||||
# analysis.scripts has all runtime hooks and all of our tools.
|
||||
# remove the other tools.
|
||||
[s for s in analysis.scripts if s[0] not in tools or s[0] == tool],
|
||||
[],
|
||||
exclude_binaries=True,
|
||||
name=tool,
|
||||
console=True,
|
||||
upx=False,
|
||||
icon='icon.ico'
|
||||
))
|
||||
|
||||
COLLECT(
|
||||
*executables,
|
||||
analysis.binaries,
|
||||
analysis.zipfiles,
|
||||
analysis.datas,
|
||||
strip=False,
|
||||
upx=False,
|
||||
name="onedir"
|
||||
)
|
||||
@ -1,10 +1,12 @@
|
||||
import io
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from release import cibuild
|
||||
|
||||
root = Path(__file__).parent.parent.parent
|
||||
|
||||
|
||||
def test_buildenviron_live():
|
||||
be = cibuild.BuildEnviron.from_env()
|
||||
@ -14,13 +16,12 @@ def test_buildenviron_live():
|
||||
def test_buildenviron_common():
|
||||
be = cibuild.BuildEnviron(
|
||||
system="Linux",
|
||||
root_dir="/foo",
|
||||
github_ref="refs/heads/master",
|
||||
root_dir=root,
|
||||
branch="master",
|
||||
)
|
||||
assert be.release_dir == os.path.join(be.root_dir, "release")
|
||||
assert be.dist_dir == os.path.join(be.root_dir, "release", "dist")
|
||||
assert be.build_dir == os.path.join(be.root_dir, "release", "build")
|
||||
assert be.is_pull_request is False
|
||||
assert be.release_dir == be.root_dir / "release"
|
||||
assert be.dist_dir == be.root_dir / "release" / "dist"
|
||||
assert be.build_dir == be.root_dir / "release" / "build"
|
||||
assert not be.has_docker_creds
|
||||
|
||||
cs = io.StringIO()
|
||||
@ -29,7 +30,7 @@ def test_buildenviron_common():
|
||||
|
||||
be = cibuild.BuildEnviron(
|
||||
system="Unknown",
|
||||
root_dir="/foo",
|
||||
root_dir=root,
|
||||
)
|
||||
with pytest.raises(cibuild.BuildError):
|
||||
be.version
|
||||
@ -37,41 +38,27 @@ def test_buildenviron_common():
|
||||
be.platform_tag
|
||||
|
||||
|
||||
def test_buildenviron_pr():
|
||||
def test_buildenviron_pr(monkeypatch):
|
||||
# Simulates a PR. We build everything, but don't have access to secret
|
||||
# credential env variables.
|
||||
be = cibuild.BuildEnviron(
|
||||
github_event_name="pull_request",
|
||||
should_build_wheel=True,
|
||||
should_build_pyinstaller=True,
|
||||
should_build_docker=True,
|
||||
)
|
||||
monkeypatch.setenv("GITHUB_REF", "refs/pull/42/merge")
|
||||
monkeypatch.setenv("CI_BUILD_WHEEL", "1")
|
||||
monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request")
|
||||
|
||||
be = cibuild.BuildEnviron.from_env()
|
||||
assert be.branch == "pr-42"
|
||||
assert be.is_pull_request
|
||||
|
||||
|
||||
def test_ci_systems():
|
||||
github = cibuild.BuildEnviron(
|
||||
github_event_name="pull_request",
|
||||
github_ref="refs/heads/master"
|
||||
)
|
||||
assert github.is_pull_request
|
||||
assert github.branch == "master"
|
||||
assert github.tag == ""
|
||||
|
||||
github2 = cibuild.BuildEnviron(
|
||||
github_event_name="pull_request",
|
||||
github_ref="refs/tags/qux"
|
||||
)
|
||||
assert github2.is_pull_request
|
||||
assert github2.branch == ""
|
||||
assert github2.tag == "qux"
|
||||
assert be.should_build_wheel
|
||||
assert not be.should_upload_pypi
|
||||
|
||||
|
||||
def test_buildenviron_commit():
|
||||
# Simulates an ordinary commit on the master branch.
|
||||
be = cibuild.BuildEnviron(
|
||||
github_ref="refs/heads/master",
|
||||
github_event_name="push",
|
||||
system="Linux",
|
||||
root_dir=root,
|
||||
branch="master",
|
||||
is_pull_request=False,
|
||||
should_build_wheel=True,
|
||||
should_build_pyinstaller=True,
|
||||
should_build_docker=True,
|
||||
@ -92,8 +79,8 @@ def test_buildenviron_releasetag():
|
||||
# Simulates a tagged release on a release branch.
|
||||
be = cibuild.BuildEnviron(
|
||||
system="Linux",
|
||||
root_dir="/foo",
|
||||
github_ref="refs/tags/v0.0.1",
|
||||
root_dir=root,
|
||||
tag="v0.0.1",
|
||||
should_build_wheel=True,
|
||||
should_build_docker=True,
|
||||
should_build_pyinstaller=True,
|
||||
@ -102,7 +89,7 @@ def test_buildenviron_releasetag():
|
||||
docker_password="bar",
|
||||
)
|
||||
assert be.tag == "v0.0.1"
|
||||
assert be.branch == ""
|
||||
assert be.branch is None
|
||||
assert be.version == "0.0.1"
|
||||
assert be.upload_dir == "0.0.1"
|
||||
assert be.docker_tag == "mitmproxy/mitmproxy:0.0.1"
|
||||
@ -116,8 +103,8 @@ def test_buildenviron_namedtag():
|
||||
# Simulates a non-release tag on a branch.
|
||||
be = cibuild.BuildEnviron(
|
||||
system="Linux",
|
||||
root_dir="/foo",
|
||||
github_ref="refs/tags/anyname",
|
||||
root_dir=root,
|
||||
tag="anyname",
|
||||
should_build_wheel=True,
|
||||
should_build_docker=True,
|
||||
should_build_pyinstaller=True,
|
||||
@ -126,7 +113,7 @@ def test_buildenviron_namedtag():
|
||||
docker_password="bar",
|
||||
)
|
||||
assert be.tag == "anyname"
|
||||
assert be.branch == ""
|
||||
assert be.branch is None
|
||||
assert be.version == "anyname"
|
||||
assert be.upload_dir == "anyname"
|
||||
assert be.docker_tag == "mitmproxy/mitmproxy:anyname"
|
||||
@ -140,8 +127,8 @@ def test_buildenviron_dev_branch():
|
||||
# Simulates a commit on a development branch on the main repo
|
||||
be = cibuild.BuildEnviron(
|
||||
system="Linux",
|
||||
root_dir="/foo",
|
||||
github_ref="refs/heads/mybranch",
|
||||
root_dir=root,
|
||||
branch="mybranch",
|
||||
should_build_wheel=True,
|
||||
should_build_docker=True,
|
||||
should_build_pyinstaller=True,
|
||||
@ -149,7 +136,7 @@ def test_buildenviron_dev_branch():
|
||||
docker_username="foo",
|
||||
docker_password="bar",
|
||||
)
|
||||
assert be.tag == ""
|
||||
assert be.tag is None
|
||||
assert be.branch == "mybranch"
|
||||
assert be.version == "mybranch"
|
||||
assert be.upload_dir == "branches/mybranch"
|
||||
@ -162,8 +149,8 @@ def test_buildenviron_maintenance_branch():
|
||||
# Simulates a commit on a release maintenance branch on the main repo
|
||||
be = cibuild.BuildEnviron(
|
||||
system="Linux",
|
||||
root_dir="/foo",
|
||||
github_ref="refs/heads/v0.x",
|
||||
root_dir=root,
|
||||
branch="v0.x",
|
||||
should_build_wheel=True,
|
||||
should_build_docker=True,
|
||||
should_build_pyinstaller=True,
|
||||
@ -171,7 +158,7 @@ def test_buildenviron_maintenance_branch():
|
||||
docker_username="foo",
|
||||
docker_password="bar",
|
||||
)
|
||||
assert be.tag == ""
|
||||
assert be.tag is None
|
||||
assert be.branch == "v0.x"
|
||||
assert be.version == "v0.x"
|
||||
assert be.upload_dir == "branches/v0.x"
|
||||
@ -180,38 +167,32 @@ def test_buildenviron_maintenance_branch():
|
||||
assert be.is_maintenance_branch
|
||||
|
||||
|
||||
def test_buildenviron_osx(tmpdir):
|
||||
def test_buildenviron_osx(tmp_path):
|
||||
be = cibuild.BuildEnviron(
|
||||
system="Darwin",
|
||||
root_dir="/foo",
|
||||
github_ref="refs/tags/v0.0.1",
|
||||
root_dir=root,
|
||||
tag="v0.0.1",
|
||||
)
|
||||
assert be.platform_tag == "osx"
|
||||
assert be.bdists == {
|
||||
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
|
||||
}
|
||||
assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-osx.tar.gz"
|
||||
assert be.archive_path == be.dist_dir / "mitmproxy-0.0.1-osx.tar.gz"
|
||||
|
||||
a = be.archive(os.path.join(tmpdir, "arch"))
|
||||
assert a
|
||||
a.close()
|
||||
with be.archive(tmp_path / "arch"):
|
||||
pass
|
||||
assert (tmp_path / "arch").exists()
|
||||
|
||||
|
||||
def test_buildenviron_windows(tmpdir):
|
||||
def test_buildenviron_windows(tmp_path):
|
||||
be = cibuild.BuildEnviron(
|
||||
system="Windows",
|
||||
root_dir="/foo",
|
||||
github_ref="refs/tags/v0.0.1",
|
||||
root_dir=root,
|
||||
tag="v0.0.1",
|
||||
)
|
||||
assert be.platform_tag == "windows"
|
||||
assert be.bdists == {
|
||||
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
|
||||
}
|
||||
assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-windows.zip"
|
||||
assert be.archive_path == be.dist_dir / "mitmproxy-0.0.1-windows.zip"
|
||||
|
||||
a = be.archive(os.path.join(tmpdir, "arch"))
|
||||
assert a
|
||||
a.close()
|
||||
with be.archive(tmp_path / "arch"):
|
||||
pass
|
||||
assert (tmp_path / "arch").exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version, tag, ok", [
|
||||
@ -229,7 +210,8 @@ def test_buildenviron_check_version(version, tag, ok, tmpdir):
|
||||
|
||||
be = cibuild.BuildEnviron(
|
||||
root_dir=tmpdir,
|
||||
github_ref=f"refs/tags/{tag}",
|
||||
system="Windows",
|
||||
tag=tag,
|
||||
)
|
||||
if ok:
|
||||
be.check_version()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user