Docker: Add aarch64 Images (#4637)

* feat(cibuild): add buildx multi arch builds

* chore: add changelog for arm64

* temporarily enable docker ci job for PRs

* Update cibuild.py

* Update cibuild.py

* chore(cibuild): create docker-container xbuilder

* chore(cibuild): fix lint

* temporarily remove run check to see error message

* Update cibuild.py

* Update cibuild.py

* Update cibuild.py

* Update main.yml

* Update main.yml

* Update main.yml

* Update cibuild.py

* Update cibuild.py

* Update Dockerfile

* cleanup #1

* next test

* move to test branch

* fixup

* now upload

* enable armv6/7

* use multi-stage build to reduce image size

* armv7?

* drop armv6/armv7

Co-authored-by: Niels Hofmans <hello@ironpeak.be>

.github/workflows/main.yml

@@ -178,7 +178,7 @@ jobs:
 
   # Separate from everything else because slow.
   build-and-deploy-docker:
-    if: github.repository == 'mitmproxy/mitmproxy' && github.ref == 'refs/heads/main'
+    if: github.repository == 'mitmproxy/mitmproxy' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dockertest')
     environment: deploy-docker
     needs:
      - test
@@ -201,6 +201,8 @@ jobs:
         with:
           name: binaries.linux
           path: release/dist
+      - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+      - uses: docker/setup-buildx-action@b1f1f719c7cd5364be7c82e366366da322d01f7c
       - run: pip install -e .[dev]
       - run: python release/cibuild.py build
       - run: python release/cibuild.py upload

CHANGELOG.md

@@ -55,6 +55,7 @@ Mitmproxy has a completely new proxy core, fixing many longstanding issues:
 * Improve readability of SHA256 fingerprint. (@wrekone)
 * Metadata and Replay Flow Filters: Flows may be filtered based on metadata and replay status. (@rbdixon)
 * Flow control: don't read connection data faster than it can be forwarded. (@hazcod)
+* Docker images for ARM64 architecture (@hazcod, @mhils)
 * Fix parsing of certificate issuer/subject with escaped special characters (@Prinzhorn)
 * Customize markers with emoji, and filters: The `flow.mark` command may be used to mark a flow with either the default
   "red ball" marker, a single character, or an emoji like `:grapes:`. Use the `~marker` filter to filter on marker

release/cibuild.py

@@ -215,7 +215,7 @@ class BuildEnviron:
     @property
     def should_upload_docker(self) -> bool:
         return all([
-            (self.is_prod_release or self.branch == "main"),
+            (self.is_prod_release or self.branch in ["main", "dockertest"]),
             self.should_build_docker,
             self.has_docker_creds,
         ])
@@ -272,22 +272,34 @@ def build_wheel(be: BuildEnviron) -> None:  # pragma: no cover
     subprocess.check_call(["tox", "-e", "wheeltest", "--", whl])
 
 
+DOCKER_PLATFORMS = "linux/amd64,linux/arm64"
+
+
 def build_docker_image(be: BuildEnviron) -> None:  # pragma: no cover
     click.echo("Building Docker images...")
 
     whl, = be.dist_dir.glob('mitmproxy-*-py3-none-any.whl')
     docker_build_dir = be.release_dir / "docker"
     shutil.copy(whl, docker_build_dir / whl.name)
+
     subprocess.check_call([
-        "docker",
+        "docker", "buildx", "build",
-        "build",
         "--tag", be.docker_tag,
+        "--platform", DOCKER_PLATFORMS,
         "--build-arg", f"MITMPROXY_WHEEL={whl.name}",
         "."
-    ],
+    ], cwd=docker_build_dir)
-        cwd=docker_build_dir
-    )
     # smoke-test the newly built docker image
+
+    # build again without --platform but with --load to make the tag available,
+    # see https://github.com/docker/buildx/issues/59#issuecomment-616050491
+    subprocess.check_call([
+        "docker", "buildx", "build",
+        "--tag", be.docker_tag,
+        "--load",
+        "--build-arg", f"MITMPROXY_WHEEL={whl.name}",
+        "."
+    ], cwd=docker_build_dir)
     r = subprocess.run([
         "docker",
         "run",
@@ -514,7 +526,20 @@ def upload():  # pragma: no cover
             "-u", be.docker_username,
             "-p", be.docker_password,
         ])
-        subprocess.check_call(["docker", "push", be.docker_tag])
+
+        whl, = be.dist_dir.glob('mitmproxy-*-py3-none-any.whl')
+        docker_build_dir = be.release_dir / "docker"
+        shutil.copy(whl, docker_build_dir / whl.name)
+        # buildx is a bit weird in that we need to reinvoke build, but oh well.
+        subprocess.check_call([
+            "docker", "buildx", "build",
+            "--tag", be.docker_tag,
+            "--push",
+            "--platform", DOCKER_PLATFORMS,
+            "--build-arg", f"MITMPROXY_WHEEL={whl.name}",
+            "."
+        ], cwd=docker_build_dir)
+
         if be.is_prod_release:
             subprocess.check_call(["docker", "tag", be.docker_tag, "mitmproxy/mitmproxy:latest"])
             subprocess.check_call(["docker", "push", "mitmproxy/mitmproxy:latest"])

release/docker/Dockerfile

@@ -1,15 +1,19 @@
-FROM python:3.9-slim-buster
+FROM python:3.9-buster as wheelbuilder
 
 ARG MITMPROXY_WHEEL
+COPY $MITMPROXY_WHEEL /wheels/
+RUN pip install wheel && pip wheel --wheel-dir /wheels /wheels/${MITMPROXY_WHEEL}
+
+FROM python:3.9-slim-buster
 
 RUN useradd -mU mitmproxy
 RUN apt-get update \
     && apt-get install -y --no-install-recommends gosu \
     && rm -rf /var/lib/apt/lists/*
 
-COPY $MITMPROXY_WHEEL /home/mitmproxy/
+COPY --from=wheelbuilder /wheels /wheels
-RUN pip3 install --no-cache-dir -U /home/mitmproxy/${MITMPROXY_WHEEL} \
+RUN pip install --no-index --find-links=/wheels mitmproxy
-    && rm -rf /home/mitmproxy/${MITMPROXY_WHEEL}
+RUN rm -rf /wheels
 
 VOLUME /home/mitmproxy/.mitmproxy