Harden CI Pipeline (#4590)

* ci: use actions/checkout@v2 * ci: always specify python version * ci: pin external actions * ci: split docs job, pin immediate dependencies * ci: correct hugo sha256sum * ci: full repo fetch depth for tests * ci: use pip-tools to pin all the things * ci: minor fixes * ci: fixup * ci: streamline pinned install * ci: minor fixes * ci: fix py3.8 pins * ci: don't persist checkout credentials * ci: always run local linter * ci: test docs deployment from actions-hardening branch * ci: fix docs job * ci: pass in credentials * ci: fix file permissions * ci: try harder to fix docs deploy * ci: fix docker artifact name * Revert "ci: test docs deployment from actions-hardening branch" This reverts commit 30cfb7a814b61a8926fc0623e3e70b6dd5106d90. * unpin PyPI dependencies * ci: install tox first * ci: fixups * ci: fixups * ci: fixups * ci: fixups
2024-11-21 22:58:24 +00:00 · 2021-05-11 11:17:09 +02:00 · 2021-05-11 11:17:09 +02:00 · 518fb94124
commit 518fb94124
parent df9dc4892b
8 changed files with 173 additions and 100 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -1,41 +1,64 @@
 name: CI

-on: [push, pull_request]
+on: [ push, pull_request ]
+
+permissions:
+  contents: read

 jobs:
  lint-pr:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v1
-      - uses: TrueBrain/actions-flake8@v1.4.1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+      - uses: TrueBrain/actions-flake8@9a43ff1b2c7b96f3edffc48a49973ce3de116ba1
+        # mirrored at https://github.com/mitmproxy/mitmproxy/settings/actions
  lint-local:
    if: github.event_name == 'push'
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
      - run: pip install tox
      - run: tox -e flake8
  filename-matching:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
      - run: pip install tox
      - run: tox -e filename_matching
  mypy:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
      - run: pip install tox
      - run: tox -e mypy
  individual-coverage:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          fetch-depth: 0
      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
      - run: pip install tox
      - run: tox -e individual_coverage
  test:
@ -54,64 +77,65 @@ jobs:
    runs-on: ${{ matrix.os }}
    steps:
      - run: printenv
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          fetch-depth: 0
      - uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.py }}
      - run: pip install tox
      - run: tox -e py
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27
+        # mirrored below and at https://github.com/mitmproxy/mitmproxy/settings/actions
        with:
          file: ./coverage.xml
          name: ${{ matrix.os }}
-  build-wheel:
-    runs-on: ubuntu-latest
-    env:
-      CI_BUILD_WHEEL: 1
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.9'
-      - run: pip install tox
-      - run: tox -e cibuild -- build
-      - uses: actions/upload-artifact@v2
-        with:
-          name: wheel
-          path: release/dist
-  build-binaries:
+
+  build:
    strategy:
      fail-fast: false
      matrix:
-        # Old Ubuntu version for old glibc
-        os: [macos-10.15, windows-2019, ubuntu-16.04]
-    runs-on: ${{ matrix.os }}
+        include:
+          - image: macos-10.15
+            platform: macos
+          - image: windows-2019
+            platform: windows
+          - image: ubuntu-16.04  # Old Ubuntu version for old glibc
+            platform: linux
+    runs-on: ${{ matrix.image }}
    env:
+      CI_BUILD_WHEEL: ${{ matrix.platform == 'linux' }}
      CI_BUILD_PYINSTALLER: 1
-      CI_BUILD_WININSTALLER: ${{ matrix.os == 'windows-2019' }}
+      CI_BUILD_WININSTALLER: ${{ matrix.platform == 'windows' }}
      CI_BUILD_KEY: ${{ secrets.CI_BUILD_KEY }}
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          fetch-depth: 0
      - uses: actions/setup-python@v2
        with:
          python-version: '3.9'
-      - if: matrix.os == 'windows-latest'
-        uses: actions/cache@v1
+      - if: matrix.platform == 'windows'
+        uses: actions/cache@v2
        with:
          path: release/installbuilder/setup
          key: installbuilder
-      - run: pip install tox
-      - run: tox -e cibuild -- build
+      - run: pip install -e .[dev]
+      - run: python release/cibuild.py build
      # artifacts must have different names, see https://github.com/actions/upload-artifact/issues/24
      - uses: actions/upload-artifact@v2
        with:
-          name: binaries.${{ matrix.os }}
+          name: binaries.${{ matrix.platform }}
          path: release/dist

  test-web-ui:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
      - run: git rev-parse --abbrev-ref HEAD
      - uses: actions/setup-node@v1
      - id: yarn-cache
@ -126,76 +150,93 @@ jobs:
        run: yarn
      - working-directory: ./web
        run: npm test
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27
+        # mirrored above and at https://github.com/mitmproxy/mitmproxy/settings/actions
        with:
          file: ./web/coverage/coverage-final.json
          name: web

  docs:
-    environment: deploy-docs
-    env:
-      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
      - uses: actions/setup-python@v2
        with:
          python-version: '3.9'
-      - run: pip install tox
      - run: |
-          wget https://github.com/gohugoio/hugo/releases/download/v0.70.0/hugo_extended_0.70.0_Linux-64bit.deb
+          wget -q https://github.com/gohugoio/hugo/releases/download/v0.83.1/hugo_extended_0.83.1_Linux-64bit.deb
+          echo "9487ea3b80f8ddd0ba600d42850b96b6a8b0bb9b41bc08cb285635ebbd41328d hugo_extended_0.83.1_Linux-64bit.deb" | sha256sum -c
          sudo dpkg -i hugo*.deb
-      - run: tox -e docs
+      - run: pip install -e .[dev]
+      - run: ./docs/build.py
+      - uses: actions/upload-artifact@v2
+        with:
+          name: docs
+          path: docs/public

  # Separate from everything else because slow.
  build-and-deploy-docker:
    if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push'
    environment: deploy-docker
-    needs: [test, test-web-ui, build-wheel]
+    needs:
+     - test
+     - test-web-ui
+     - build
+     - docs
    runs-on: ubuntu-latest
    env:
      CI_BUILD_DOCKER: 1
      DOCKER_USERNAME: mitmbot
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
      - uses: actions/setup-python@v2
        with:
          python-version: '3.9'
-      - run: pip install tox
      - uses: actions/download-artifact@v2
        with:
-          name: wheel
+          name: binaries.linux
          path: release/dist
-      - run: tox -e cibuild -- build
-      - run: tox -e cibuild -- upload
+      - run: pip install -e .[dev]
+      - run: python release/cibuild.py build
+      - run: python release/cibuild.py upload

  deploy:
+    # This action has access to our AWS keys, so we are extra careful here.
+    # In particular, we don't blindly `pip install` anything to minimize the risk of supply chain attacks.
    if: github.repository == 'mitmproxy/mitmproxy' && github.event_name == 'push'
    environment: deploy
+    needs:
+      - test
+      - test-web-ui
+      - build
+      - docs
    runs-on: ubuntu-latest
-    needs: [test, test-web-ui, build-wheel, build-binaries]
    env:
-      CI_BUILD_WHEEL: 1
-      CI_BUILD_PYINSTALLER: 1
-      CI_BUILD_WININSTALLER: 1
      TWINE_USERNAME: mitmproxy
      TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: us-west-2
    steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
      - uses: actions/setup-python@v2
        with:
          python-version: '3.9'
+      - run: sudo apt-get install -y twine awscli
      - uses: actions/download-artifact@v2
        with:
          path: release/dist
+      - run: mv release/dist/docs docs/public
      # move artifacts from their subfolders into release/dist
      - run: find release/dist -mindepth 2 -type f -exec mv {} release/dist \;
      # and then delete the empty folders
      - run: find release/dist -type d -empty -delete
      - run: ls release/dist
-      - run: pip install tox
-      - run: tox -e cibuild -- upload
+      - run: ./release/deploy.py
--- a/docs/build.py
+++ b/docs/build.py
@ -8,7 +8,7 @@ here = Path(__file__).parent

 for script in sorted((here / "scripts").glob("*.py")):
    print(f"Generating output for {script.name}...")
-    out = subprocess.check_output(["python3", script.absolute()], text=True)
+    out = subprocess.check_output(["python3", script.absolute()], cwd=here, text=True)
    if out:
        (here / "src" / "generated" / f"{script.stem}.html").write_text(out, encoding="utf8")

--- a/docs/ci.sh
+++ b/docs/ci.sh
@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o pipefail
-# set -o xtrace
-
-# This script gets run from CI to render and upload docs for the main branch.
-
-./build.py
-
-# Only upload if we have defined credentials - we only have these defined for
-# trusted commits (i.e. not PRs).
-if [[ -n "${AWS_ACCESS_KEY_ID}" && $GITHUB_REF == "refs/heads/main" ]]; then
-    aws s3 sync --delete --acl public-read ./public s3://docs.mitmproxy.org/dev
-    aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
-        --paths "/dev/*"
-fi
--- a/release/cibuild.py
+++ b/release/cibuild.py
@ -1,6 +1,7 @@
 #!/usr/bin/env python3

 import contextlib
+import hashlib
 import os
 import platform
 import re
@ -382,6 +383,7 @@ def build_wininstaller(be: BuildEnviron) -> None:  # pragma: no cover
    click.echo("Building wininstaller package...")

    IB_VERSION = "20.12.0"
+    IB_SETUP_SHA256 = "657f4785c7d70f140468435b99e79ced813e7e051106e7525e0c819efffb40d3"
    IB_DIR = be.release_dir / "installbuilder"
    IB_SETUP = IB_DIR / "setup" / f"{IB_VERSION}-installer.exe"
    IB_CLI = Path(fr"C:\Program Files\VMware InstallBuilder Enterprise {IB_VERSION}\bin\builder-cli.exe")
@ -408,6 +410,16 @@ def build_wininstaller(be: BuildEnviron) -> None:  # pragma: no cover
            )
            tmp.rename(IB_SETUP)

+        ib_setup_hash = hashlib.sha256()
+        with IB_SETUP.open("rb") as fp:
+            while True:
+                data = fp.read(65_536)
+                if not data:
+                    break
+                ib_setup_hash.update(data)
+        if ib_setup_hash.hexdigest() != IB_SETUP_SHA256:  # pragma: no cover
+            raise RuntimeError("InstallBuilder hashes don't match.")
+
        click.echo("Install InstallBuilder...")
        subprocess.run([IB_SETUP, "--mode", "unattended", "--unattendedmodeui", "none"], check=True)
        assert IB_CLI.is_file()
--- a/release/deploy.py
+++ b/release/deploy.py
@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+if __name__ == "__main__":
+    ref = os.environ["GITHUB_REF"]
+    branch: Optional[str] = None
+    tag: Optional[str] = None
+    if ref.startswith("refs/heads/"):
+        branch = ref.replace("refs/heads/", "")
+    elif ref.startswith("refs/tags/"):
+        tag = ref.replace("refs/tags/", "")
+    else:
+        raise AssertionError
+
+    # Upload binaries (be it release or snapshot)
+    if tag:
+        upload_dir = tag
+    else:
+        upload_dir = f"branches/{branch}"
+    subprocess.check_call([
+        "aws", "s3", "cp",
+        "--acl", "public-read",
+        f"./release/dist/",
+        f"s3://snapshots.mitmproxy.org/{upload_dir}/",
+        "--recursive",
+    ])
+
+    # Upload releases to PyPI
+    if tag:
+        whl, = Path("release/dist/").glob('mitmproxy-*-py3-none-any.whl')
+        subprocess.check_call(["twine", "upload", whl])
+
+    # Upload dev docs
+    if branch == "main" or branch == "actions-hardening":  # FIXME remove
+        subprocess.check_call([
+            "aws", "configure",
+            "set", "preview.cloudfront", "true"
+        ])
+        subprocess.check_call([
+            "aws", "s3",
+            "sync",
+            "--delete",
+            "--acl", "public-read",
+            "docs/public",
+            "s3://docs.mitmproxy.org/dev"
+        ])
+        subprocess.check_call([
+            "aws", "cloudfront",
+            "create-invalidation",
+            "--distribution-id", "E1TH3USJHFQZ5Q",
+            "--paths", "/dev/*"
+        ])
--- a/setup.py
+++ b/setup.py
@ -1,8 +1,8 @@
 import os
+import re
 from codecs import open

-import re
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup

 # Based on https://github.com/pypa/sampleproject/blob/main/setup.py
 # and https://python-packaging-user-guide.readthedocs.org/
@ -100,6 +100,7 @@ setup(
            "hypothesis>=5.8,<6.11",
            "parver>=0.1,<2.0",
            "pdoc>=4.0.0",
+            "pyinstaller==4.3",
            "pytest-asyncio>=0.10.0,<0.14,!=0.14",
            "pytest-cov>=2.7.1,<3",
            "pytest-timeout>=1.3.3,<2",
@ -107,6 +108,7 @@ setup(
            "pytest>=6.1.0,<7",
            "requests>=2.9.1,<3",
            "tox>=3.5,<4",
-        ]
+            "wheel>=0.36.2,<0.37"
+        ],
    }
 )
--- a/tox.ini
+++ b/tox.ini
@ -36,17 +36,6 @@ commands =
 commands =
  python ./test/individual_coverage.py {posargs}

-[testenv:cibuild]
-passenv = CI_* GITHUB_* AWS_* TWINE_* DOCKER_*
-deps =
-  -e .[dev]
-  pyinstaller==4.3
-  twine==3.4.1
-  awscli
-commands =
-  mitmdump --version
-  python ./release/cibuild.py {posargs}
-
 [testenv:wheeltest]
 recreate = True
 deps =
@ -55,12 +44,3 @@ commands =
  mitmproxy --version
  mitmdump --version
  mitmweb --version
-
-[testenv:docs]
-passenv = GITHUB_* AWS_*
-deps =
-  -e .[dev]
-  awscli
-changedir = docs
-commands =
-  ./ci.sh