Harden CI Pipeline (#4590)

* ci: use actions/checkout@v2

* ci: always specify python version

* ci: pin external actions

* ci: split docs job, pin immediate dependencies

* ci: correct hugo sha256sum

* ci: full repo fetch depth for tests

* ci: use pip-tools to pin all the things

* ci: minor fixes

* ci: fixup

* ci: streamline pinned install

* ci: minor fixes

* ci: fix py3.8 pins

* ci: don't persist checkout credentials

* ci: always run local linter

* ci: test docs deployment from actions-hardening branch

* ci: fix docs job

* ci: pass in credentials

* ci: fix file permissions

* ci: try harder to fix docs deploy

* ci: fix docker artifact name

* Revert "ci: test docs deployment from actions-hardening branch"

This reverts commit 30cfb7a814b61a8926fc0623e3e70b6dd5106d90.

* unpin PyPI dependencies

* ci: install tox first

* ci: fixups

* ci: fixups

* ci: fixups

* ci: fixups

.github/workflows/main.yml

@@ -1,41 +1,64 @@
 name: CI
 
-on: [push, pull_request]
+on: [ push, pull_request ]
+
+permissions:
+  contents: read
 
 jobs:
   lint-pr:
     if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
-      - uses: TrueBrain/actions-flake8@v1.4.1
+        with:
+          persist-credentials: false
+      - uses: TrueBrain/actions-flake8@9a43ff1b2c7b96f3edffc48a49973ce3de116ba1
+        # mirrored at https://github.com/mitmproxy/mitmproxy/settings/actions
   lint-local:
     if: github.event_name == 'push'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
       - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
       - run: pip install tox
       - run: tox -e flake8
   filename-matching:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
       - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
       - run: pip install tox
       - run: tox -e filename_matching
   mypy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
       - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
       - run: pip install tox
       - run: tox -e mypy
   individual-coverage:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          fetch-depth: 0
       - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
       - run: pip install tox
       - run: tox -e individual_coverage
   test:
@@ -54,64 +77,65 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - run: printenv
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          fetch-depth: 0
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.py }}
       - run: pip install tox
       - run: tox -e py
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27
+        # mirrored below and at https://github.com/mitmproxy/mitmproxy/settings/actions
         with:
           file: ./coverage.xml
           name: ${{ matrix.os }}
-  build-wheel:
+
-    runs-on: ubuntu-latest
+  build:
-    env:
-      CI_BUILD_WHEEL: 1
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.9'
-      - run: pip install tox
-      - run: tox -e cibuild -- build
-      - uses: actions/upload-artifact@v2
-        with:
-          name: wheel
-          path: release/dist
-  build-binaries:
     strategy:
       fail-fast: false
       matrix:
-        # Old Ubuntu version for old glibc
+        include:
-        os: [macos-10.15, windows-2019, ubuntu-16.04]
+          - image: macos-10.15
-    runs-on: ${{ matrix.os }}
+            platform: macos
+          - image: windows-2019
+            platform: windows
+          - image: ubuntu-16.04  # Old Ubuntu version for old glibc
+            platform: linux
+    runs-on: ${{ matrix.image }}
     env:
+      CI_BUILD_WHEEL: ${{ matrix.platform == 'linux' }}
       CI_BUILD_PYINSTALLER: 1
-      CI_BUILD_WININSTALLER: ${{ matrix.os == 'windows-2019' }}
+      CI_BUILD_WININSTALLER: ${{ matrix.platform == 'windows' }}
       CI_BUILD_KEY: ${{ secrets.CI_BUILD_KEY }}
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          fetch-depth: 0
       - uses: actions/setup-python@v2
         with:
           python-version: '3.9'
-      - if: matrix.os == 'windows-latest'
+      - if: matrix.platform == 'windows'
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: release/installbuilder/setup
           key: installbuilder
-      - run: pip install tox
+      - run: pip install -e .[dev]
-      - run: tox -e cibuild -- build
+      - run: python release/cibuild.py build
       # artifacts must have different names, see https://github.com/actions/upload-artifact/issues/24
       - uses: actions/upload-artifact@v2
         with:
-          name: binaries.${{ matrix.os }}
+          name: binaries.${{ matrix.platform }}
           path: release/dist
 
   test-web-ui:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
       - run: git rev-parse --abbrev-ref HEAD
       - uses: actions/setup-node@v1
       - id: yarn-cache
@@ -126,76 +150,93 @@ jobs:
         run: yarn
       - working-directory: ./web
         run: npm test
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27
+        # mirrored above and at https://github.com/mitmproxy/mitmproxy/settings/actions
         with:
           file: ./web/coverage/coverage-final.json
           name: web
 
   docs:
-    environment: deploy-docs
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
       - uses: actions/setup-python@v2
         with:
           python-version: '3.9'
-      - run: pip install tox
       - run: |
-          wget https://github.com/gohugoio/hugo/releases/download/v0.70.0/hugo_extended_0.70.0_Linux-64bit.deb
+          wget -q https://github.com/gohugoio/hugo/releases/download/v0.83.1/hugo_extended_0.83.1_Linux-64bit.deb
+          echo "9487ea3b80f8ddd0ba600d42850b96b6a8b0bb9b41bc08cb285635ebbd41328d hugo_extended_0.83.1_Linux-64bit.deb" | sha256sum -c
           sudo dpkg -i hugo*.deb
-      - run: tox -e docs
+      - run: pip install -e .[dev]
+      - run: ./docs/build.py
+      - uses: actions/upload-artifact@v2
+        with:
+          name: docs
+          path: docs/public
 
   # Separate from everything else because slow.
   build-and-deploy-docker:
     if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push'
     environment: deploy-docker
-    needs: [test, test-web-ui, build-wheel]
+    needs:
+     - test
+     - test-web-ui
+     - build
+     - docs
     runs-on: ubuntu-latest
     env:
       CI_BUILD_DOCKER: 1
       DOCKER_USERNAME: mitmbot
       DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
       - uses: actions/setup-python@v2
         with:
           python-version: '3.9'
-      - run: pip install tox
       - uses: actions/download-artifact@v2
         with:
-          name: wheel
+          name: binaries.linux
           path: release/dist
-      - run: tox -e cibuild -- build
+      - run: pip install -e .[dev]
-      - run: tox -e cibuild -- upload
+      - run: python release/cibuild.py build
+      - run: python release/cibuild.py upload
 
   deploy:
+    # This action has access to our AWS keys, so we are extra careful here.
+    # In particular, we don't blindly `pip install` anything to minimize the risk of supply chain attacks.
     if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push'
     environment: deploy
+    needs:
+      - test
+      - test-web-ui
+      - build
+      - docs
     runs-on: ubuntu-latest
-    needs: [test, test-web-ui, build-wheel, build-binaries]
     env:
-      CI_BUILD_WHEEL: 1
-      CI_BUILD_PYINSTALLER: 1
-      CI_BUILD_WININSTALLER: 1
       TWINE_USERNAME: mitmproxy
       TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-west-2
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
       - uses: actions/setup-python@v2
         with:
           python-version: '3.9'
+      - run: sudo apt-get install -y twine awscli
       - uses: actions/download-artifact@v2
         with:
           path: release/dist
+      - run: mv release/dist/docs docs/public
       # move artifacts from their subfolders into release/dist
       - run: find release/dist -mindepth 2 -type f -exec mv {} release/dist \;
       # and then delete the empty folders
       - run: find release/dist -type d -empty -delete
       - run: ls release/dist
-      - run: pip install tox
+      - run: ./release/deploy.py
-      - run: tox -e cibuild -- upload

docs/build.py

@@ -8,7 +8,7 @@ here = Path(__file__).parent
 
 for script in sorted((here / "scripts").glob("*.py")):
     print(f"Generating output for {script.name}...")
-    out = subprocess.check_output(["python3", script.absolute()], text=True)
+    out = subprocess.check_output(["python3", script.absolute()], cwd=here, text=True)
     if out:
         (here / "src" / "generated" / f"{script.stem}.html").write_text(out, encoding="utf8")
 

docs/ci.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o pipefail
-# set -o xtrace
-
-# This script gets run from CI to render and upload docs for the main branch.
-
-./build.py
-
-# Only upload if we have defined credentials - we only have these defined for
-# trusted commits (i.e. not PRs).
-if [[ -n "${AWS_ACCESS_KEY_ID}" && $GITHUB_REF == "refs/heads/main" ]]; then
-    aws s3 sync --delete --acl public-read ./public s3://docs.mitmproxy.org/dev
-    aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
-        --paths "/dev/*"
-fi

release/cibuild.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import contextlib
+import hashlib
 import os
 import platform
 import re
@@ -382,6 +383,7 @@ def build_wininstaller(be: BuildEnviron) -> None:  # pragma: no cover
     click.echo("Building wininstaller package...")
 
     IB_VERSION = "20.12.0"
+    IB_SETUP_SHA256 = "657f4785c7d70f140468435b99e79ced813e7e051106e7525e0c819efffb40d3"
     IB_DIR = be.release_dir / "installbuilder"
     IB_SETUP = IB_DIR / "setup" / f"{IB_VERSION}-installer.exe"
     IB_CLI = Path(fr"C:\Program Files\VMware InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe")
@@ -408,6 +410,16 @@ def build_wininstaller(be: BuildEnviron) -> None:  # pragma: no cover
             )
             tmp.rename(IB_SETUP)
 
+        ib_setup_hash = hashlib.sha256()
+        with IB_SETUP.open("rb") as fp:
+            while True:
+                data = fp.read(65_536)
+                if not data:
+                    break
+                ib_setup_hash.update(data)
+        if ib_setup_hash.hexdigest() != IB_SETUP_SHA256:  # pragma: no cover
+            raise RuntimeError("InstallBuilder hashes don't match.")
+
         click.echo("Install InstallBuilder...")
         subprocess.run([IB_SETUP, "--mode", "unattended", "--unattendedmodeui", "none"], check=True)
         assert IB_CLI.is_file()

release/deploy.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+if __name__ == "__main__":
+    ref = os.environ["GITHUB_REF"]
+    branch: Optional[str] = None
+    tag: Optional[str] = None
+    if ref.startswith("refs/heads/"):
+        branch = ref.replace("refs/heads/", "")
+    elif ref.startswith("refs/tags/"):
+        tag = ref.replace("refs/tags/", "")
+    else:
+        raise AssertionError
+
+    # Upload binaries (be it release or snapshot)
+    if tag:
+        upload_dir = tag
+    else:
+        upload_dir = f"branches/{branch}"
+    subprocess.check_call([
+        "aws", "s3", "cp",
+        "--acl", "public-read",
+        f"./release/dist/",
+        f"s3://snapshots.mitmproxy.org/{upload_dir}/",
+        "--recursive",
+    ])
+
+    # Upload releases to PyPI
+    if tag:
+        whl, = Path("release/dist/").glob('mitmproxy-*-py3-none-any.whl')
+        subprocess.check_call(["twine", "upload", whl])
+
+    # Upload dev docs
+    if branch == "main" or branch == "actions-hardening":  # FIXME remove
+        subprocess.check_call([
+            "aws", "configure",
+            "set", "preview.cloudfront", "true"
+        ])
+        subprocess.check_call([
+            "aws", "s3",
+            "sync",
+            "--delete",
+            "--acl", "public-read",
+            "docs/public",
+            "s3://docs.mitmproxy.org/dev"
+        ])
+        subprocess.check_call([
+            "aws", "cloudfront",
+            "create-invalidation",
+            "--distribution-id", "E1TH3USJHFQZ5Q",
+            "--paths", "/dev/*"
+        ])

setup.py

@@ -1,8 +1,8 @@
 import os
+import re
 from codecs import open
 
-import re
+from setuptools import find_packages, setup
-from setuptools import setup, find_packages
 
 # Based on https://github.com/pypa/sampleproject/blob/main/setup.py
 # and https://python-packaging-user-guide.readthedocs.org/
@@ -100,6 +100,7 @@ setup(
             "hypothesis>=5.8,<6.11",
             "parver>=0.1,<2.0",
             "pdoc>=4.0.0",
+            "pyinstaller==4.3",
             "pytest-asyncio>=0.10.0,<0.14,!=0.14",
             "pytest-cov>=2.7.1,<3",
             "pytest-timeout>=1.3.3,<2",
@@ -107,6 +108,7 @@ setup(
             "pytest>=6.1.0,<7",
             "requests>=2.9.1,<3",
             "tox>=3.5,<4",
-        ]
+            "wheel>=0.36.2,<0.37"
+        ],
     }
 )

tox.ini

@@ -36,17 +36,6 @@ commands =
 commands =
   python ./test/individual_coverage.py {posargs}
 
-[testenv:cibuild]
-passenv = CI_* GITHUB_* AWS_* TWINE_* DOCKER_*
-deps =
-  -e .[dev]
-  pyinstaller==4.3
-  twine==3.4.1
-  awscli
-commands =
-  mitmdump --version
-  python ./release/cibuild.py {posargs}
-
 [testenv:wheeltest]
 recreate = True
 deps =
@@ -55,12 +44,3 @@ commands =
   mitmproxy --version
   mitmdump --version
   mitmweb --version
-
-[testenv:docs]
-passenv = GITHUB_* AWS_*
-deps =
-  -e .[dev]
-  awscli
-changedir = docs
-commands =
-  ./ci.sh