add docker to the main repository

2024-11-22 15:37:45 +00:00 · 2018-05-18 10:37:56 +02:00 · 2018-05-18 10:37:56 +02:00 · 46960fa080
commit 46960fa080
parent eaca8fdaf7
8 changed files with 214 additions and 84 deletions
--- a/.appveyor.yml
+++ b/.appveyor.yml
@ -11,6 +11,7 @@ environment:
  matrix:
    - PYTHON: "C:\\Python36"
      TOXENV: "py36"
+      PYINSTALLER: "1"

 install:
  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
@ -21,7 +22,6 @@ test_script:
  - ps: "tox -- --verbose --cov-report=term"
  - ps: |
      $Env:VERSION = $(python -m mitmproxy.version)
-      $Env:SKIP_MITMPROXY = "python -c `"print('skip mitmproxy')`""
      tox -e cibuild -- build
  - ps: |
      if($Env:RTOOL_KEY) {
--- a/.travis.yml
+++ b/.travis.yml
@ -21,9 +21,12 @@ matrix:
    - os: osx
      osx_image: xcode7.3
      language: generic
-      env: TOXENV=py36 BDIST=1
+      env: TOXENV=py36 CIBUILD=1 PYINSTALLER=1
    - python: 3.6
-      env: TOXENV=py36 BDIST=1 WHEEL=1
+      env: TOXENV=py36 CIBUILD=1 PYINSTALLER=1 WHEEL=1 DOCKER=1
+      sudo: required
+      services:
+        - docker
    - python: 3.6
      env: TOXENV=individual_coverage
    - python: "3.7-dev"
@ -67,19 +70,16 @@ install:
  - pip install tox virtualenv setuptools

 script:
+  # All these steps MUST succeed for the job to be successful!
+  # Using the after_success block DOES NOT capture errors.
+  # Pull requests might break our build - we need to check this.
+  # Pull requests are not allowed to upload build artifacts - enforced by cibuild script.
  - tox -- --verbose --cov-report=term
  - |
-    if [[ $BDIST == "1" ]]
+    if [[ $CIBUILD == "1" ]]
    then
        git fetch --unshallow --tags
-        tox -e cibuild -- build
-    fi
-
-after_success:
-  - |
-    if [[ $BDIST == "1" ]]
-    then
-        tox -e cibuild -- upload
+        tox -e cibuild -- build && tox -e cibuild -- upload
    fi

 notifications:
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -0,0 +1,41 @@
+FROM alpine:3.7
+
+ENV LANG=en_US.UTF-8
+
+ARG WHEEL_MITMPROXY
+ARG WHEEL_BASENAME_MITMPROXY
+
+COPY $WHEEL_MITMPROXY /home/mitmproxy/
+
+# Add our user first to make sure the ID get assigned consistently,
+# regardless of whatever dependencies get added.
+RUN addgroup -S mitmproxy && adduser -S -G mitmproxy mitmproxy \
+    && apk add --no-cache \
+        su-exec \
+        git \
+        g++ \
+        libffi \
+        libffi-dev \
+        libstdc++ \
+        openssl \
+        openssl-dev \
+        python3 \
+        python3-dev \
+    && python3 -m ensurepip \
+    && LDFLAGS=-L/lib pip3 install -U /home/mitmproxy/${WHEEL_BASENAME_MITMPROXY} \
+    && apk del --purge \
+        git \
+        g++ \
+        libffi-dev \
+        openssl-dev \
+        python3-dev \
+    && rm -rf ~/.cache/pip /home/mitmproxy/${WHEEL_BASENAME_MITMPROXY}
+
+VOLUME /home/mitmproxy/.mitmproxy
+
+COPY docker/docker-entrypoint.sh /usr/local/bin/
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+EXPOSE 8080 8081
+
+CMD ["mitmproxy"]
--- a/docker/README.md
+++ b/docker/README.md
@ -0,0 +1,38 @@
+# mitmproxy
+
+Containerized version of [mitmproxy](https://mitmproxy.org/), an interactive SSL-capable intercepting HTTP proxy.
+
+# Usage
+
+```sh
+$ docker run --rm -it [-v ~/.mitmproxy:/home/mitmproxy/.mitmproxy] -p 8080:8080 mitmproxy/mitmproxy
+```
+The *volume mount* is optional: It's to store the generated CA certificates.
+
+Once started, mitmproxy listens as a HTTP proxy on `localhost:8080`:
+```sh
+$ http_proxy=http://localhost:8080/ curl http://example.com/
+$ https_proxy=http://localhost:8080/ curl -k https://example.com/
+```
+
+You can also start `mitmdump` by just adding that to the end of the command-line:
+```sh
+$ docker run --rm -it -p 8080:8080 mitmproxy/mitmproxy mitmdump
+```
+
+For `mitmweb`, you also need to expose port 8081:
+```sh
+# this makes :8081 accessible to the local machine only
+$ docker run --rm -it -p 8080:8080 -p 127.0.0.1:8081:8081 mitmproxy/mitmproxy mitmweb --web-iface 0.0.0.0
+```
+
+You can also pass options directly via the CLI:
+```sh
+$ docker run --rm -it -p 8080:8080 mitmproxy/mitmproxy mitmdump --set ssl_insecure=true
+```
+
+For further details, please consult the mitmproxy [documentation](http://docs.mitmproxy.org/en/stable/).
+
+# Tags
+
+The available release tags can be seen [here](https://hub.docker.com/r/mitmproxy/mitmproxy/tags/).
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@ -0,0 +1,13 @@
+#!/bin/sh
+set -e
+
+MITMPROXY_PATH="/home/mitmproxy/.mitmproxy"
+
+if [[ "$1" = "mitmdump" || "$1" = "mitmproxy" || "$1" = "mitmweb" ]]; then
+        mkdir -p "$MITMPROXY_PATH"
+        chown -R mitmproxy:mitmproxy "$MITMPROXY_PATH"
+
+        su-exec mitmproxy "$@"
+else
+        exec "$@"
+fi
--- a/release/README.md
+++ b/release/README.md
@ -37,22 +37,10 @@ release for! The command examples assume that you have a git remote called
  `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/v<version number here>`

 ## Docker
- Update docker-releases repo
-  - Create a new branch based of master for major versions.
-  - Update the dependencies in [alpine/requirements.txt](https://github.com/mitmproxy/docker-releases/commit/3d6a9989fde068ad0aea257823ac3d7986ff1613#diff-9b7e0eea8ae74688b1ac13ea080549ba)
-    * Creating a fresh venv, pip-installing the new wheel in there, and then export all packages:
-    * `virtualenv -ppython3.6 venv && source venv/bin/activate && pip install mitmproxy && pip freeze`
-  - Tag the commit with the correct version
-    * `v2.0.0` for new major versions
-    * `v2.0.2` for new patch versions
- Update `latest` tag [here](https://hub.docker.com/r/mitmproxy/mitmproxy/~/settings/automated-builds/)
- Check that the build for this tag succeeds [https://hub.docker.com/r/mitmproxy/mitmproxy/builds/](here)
- If build failed:
-  - Fix it and commit
-  - `git tag 3.0.2` the new commit
-  - `git push origin :refs/tags/3.0.2` to delete the old remote tag
-  - `git push --tags` to push the new tag
-  - Check the build details page again
+- The docker image is built on Travis and pushed to Docker Hub automatically.
+- Please check https://hub.docker.com/r/mitmproxy/mitmproxy/tags/ about the latest version
+- Update `latest` tag: TODO write instructions
+

 ## Website
 - Update version here:
--- a/release/cibuild.py
+++ b/release/cibuild.py
@ -13,18 +13,7 @@ import zipfile
 from os.path import join, abspath, dirname, exists, basename

 import click
-
-# https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
-# scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
-if platform.system() == "Windows":
-    VENV_BIN = "Scripts"
-    PYINSTALLER_ARGS = [
-        # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
-        "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
-    ]
-else:
-    VENV_BIN = "bin"
-    PYINSTALLER_ARGS = []
+import cryptography.fernet

 # ZipFile and tarfile have slightly different APIs. Fix that.
 if platform.system() == "Windows":
@ -44,20 +33,9 @@ PLATFORM_TAG = {

 ROOT_DIR = abspath(join(dirname(__file__), ".."))
 RELEASE_DIR = join(ROOT_DIR, "release")
-
 BUILD_DIR = join(RELEASE_DIR, "build")
 DIST_DIR = join(RELEASE_DIR, "dist")

-PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
-# PyInstaller 3.2 does not bundle pydivert's Windivert binaries
-PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
-PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
-PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
-
-VENV_DIR = join(BUILD_DIR, "venv")
-
-# Project Configuration
-VERSION_FILE = join(ROOT_DIR, "mitmproxy", "version.py")
 BDISTS = {
    "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
    "pathod": ["pathoc", "pathod"]
@ -83,6 +61,14 @@ else:
    print("Could not establish build name - exiting." % BRANCH)
    sys.exit(0)

+print("BUILD PLATFORM_TAG=%s" % PLATFORM_TAG)
+print("BUILD ROOT_DIR=%s" % ROOT_DIR)
+print("BUILD RELEASE_DIR=%s" % RELEASE_DIR)
+print("BUILD BUILD_DIR=%s" % BUILD_DIR)
+print("BUILD DIST_DIR=%s" % DIST_DIR)
+print("BUILD BDISTS=%s" % BDISTS)
+print("BUILD TAG=%s" % TAG)
+print("BUILD BRANCH=%s" % BRANCH)
 print("BUILD VERSION=%s" % VERSION)
 print("BUILD UPLOAD_DIR=%s" % UPLOAD_DIR)

@ -116,22 +102,28 @@ def cli():
    pass


-@cli.command("info")
-def info():
-    click.echo("Version: %s" % VERSION)
-
-
@cli.command("build")
 def build():
    """
    Build a binary distribution
    """
    os.makedirs(DIST_DIR, exist_ok=True)
+
    if "WHEEL" in os.environ:
-        build_wheel()
+        whl = build_wheel()
    else:
        click.echo("Not building wheels.")
-    build_pyinstaller()
+
+    if "WHEEL" in os.environ and "DOCKER" in os.environ:
+        # Docker image requires wheels
+        build_docker_image(whl)
+    else:
+        click.echo("Not building Docker image.")
+
+    if "PYINSTALLER" in os.environ:
+        build_pyinstaller()
+    else:
+        click.echo("Not building PyInstaller packages.")


 def build_wheel():
@ -154,8 +146,38 @@ def build_wheel():
        whl
    ])

+    return whl
+
+
+def build_docker_image(whl):
+    click.echo("Building Docker image...")
+    subprocess.check_call([
+        "docker",
+        "build",
+        "--build-arg", "WHEEL_MITMPROXY={}".format(os.path.relpath(whl, ROOT_DIR)),
+        "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(basename(whl)),
+        "--file", "docker/Dockerfile",
+        "."
+    ])
+

 def build_pyinstaller():
+    PYINSTALLER_SPEC = join(RELEASE_DIR, "specs")
+    # PyInstaller 3.2 does not bundle pydivert's Windivert binaries
+    PYINSTALLER_HOOKS = join(RELEASE_DIR, "hooks")
+    PYINSTALLER_TEMP = join(BUILD_DIR, "pyinstaller")
+    PYINSTALLER_DIST = join(BUILD_DIR, "binaries", PLATFORM_TAG)
+
+    # https://virtualenv.pypa.io/en/latest/userguide.html#windows-notes
+    # scripts and executables on Windows go in ENV\Scripts\ instead of ENV/bin/
+    if platform.system() == "Windows":
+        PYINSTALLER_ARGS = [
+            # PyInstaller < 3.2 does not handle Python 3.5's ucrt correctly.
+            "-p", r"C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86",
+        ]
+    else:
+        PYINSTALLER_ARGS = []
+
    if exists(PYINSTALLER_TEMP):
        shutil.rmtree(PYINSTALLER_TEMP)
    if exists(PYINSTALLER_DIST):
@ -170,7 +192,7 @@ def build_pyinstaller():
                # This is PyInstaller, so it messes up paths.
                # We need to make sure that we are in the spec folder.
                with chdir(PYINSTALLER_SPEC):
-                    click.echo("Building %s binary..." % tool)
+                    click.echo("Building PyInstaller %s binary..." % tool)
                    excludes = []
                    if tool != "mitmweb":
                        excludes.append("mitmproxy.tools.web")
@ -218,28 +240,25 @@ def build_pyinstaller():
        click.echo("Packed {}.".format(archive_name(bdist)))


-def is_pr():
-    if "TRAVIS_PULL_REQUEST" in os.environ:
-        if os.environ["TRAVIS_PULL_REQUEST"] == "false":
-            return False
-        return True
-    elif os.environ.get("APPVEYOR_PULL_REQUEST_NUMBER"):
-        return True
-    return False
-
-
@cli.command("upload")
 def upload():
    """
-        Upload build artifacts to snapshot server and
-        upload wheel package to PyPi
+        Upload build artifacts
+
+        Uploads the wheels package to PyPi.
+        Uploads the Pyinstaller and wheels packages to the snapshot server.
+        Pushes the Docker image to Docker Hub.
    """
-    # This requires some explanation. The AWS access keys are only exposed to
-    # privileged builds - that is, they are not available to PRs from forks.
-    # However, they ARE exposed to PRs from a branch within the main repo. This
-    # check catches that corner case, and prevents an inadvertent upload.
-    if is_pr():
-        click.echo("Refusing to upload a pull request")
+
+    # Our credentials are only available from within the main repository and not forks.
+    # We need to prevent uploads from all BUT the branches in the main repository.
+    # Pull requests and master-branches of forks are not allowed to upload.
+    is_pull_request = (
+        ("TRAVIS_PULL_REQUEST" in os.environ and os.environ["TRAVIS_PULL_REQUEST"] != "false") or
+        "APPVEYOR_PULL_REQUEST_NUMBER" in os.environ
+    )
+    if is_pull_request:
+        click.echo("Refusing to upload artifacts from a pull request!")
        return

    if "AWS_ACCESS_KEY_ID" in os.environ:
@ -247,7 +266,7 @@ def upload():
            "aws", "s3", "cp",
            "--acl", "public-read",
            DIST_DIR + "/",
-            "s3://snapshots.mitmproxy.org/%s/" % UPLOAD_DIR,
+            "s3://snapshots.mitmproxy.org/{}/".format(UPLOAD_DIR),
            "--recursive",
        ])

@ -258,14 +277,45 @@ def upload():
        "TWINE_PASSWORD" in os.environ
    )
    if upload_pypi:
-        filename = "mitmproxy-{version}-py3-none-any.whl".format(version=VERSION)
-        click.echo("Uploading {} to PyPi...".format(filename))
+        whl = glob.glob(join(DIST_DIR, 'mitmproxy-*-py3-none-any.whl'))[0]
+        click.echo("Uploading {} to PyPi...".format(whl))
        subprocess.check_call([
            "twine",
            "upload",
-            join(DIST_DIR, filename)
+            whl
        ])

+    upload_docker = (
+        (TAG or BRANCH == "master") and
+        "DOCKER" in os.environ and
+        "DOCKER_USERNAME" in os.environ and
+        "DOCKER_PASSWORD" in os.environ
+    )
+    if upload_docker:
+        docker_tag = "dev" if BRANCH == "master" else VERSION
+
+        click.echo("Uploading Docker image to tag={}...".format(docker_tag))
+        subprocess.check_call([
+            "docker",
+            "login",
+            "-u", os.environ["DOCKER_USERNAME"],
+            "-p", os.environ["DOCKER_PASSWORD"],
+        ])
+        subprocess.check_call([
+            "docker",
+            "push",
+            "mitmproxy/mitmproxy:{}".format(docker_tag),
+        ])
+
+
+@cli.command("decrypt")
+@click.argument('infile', type=click.File('rb'))
+@click.argument('outfile', type=click.File('wb'))
+@click.argument('key', envvar='RTOOL_KEY')
+def decrypt(infile, outfile, key):
+    f = cryptography.fernet.Fernet(key.encode())
+    outfile.write(f.decrypt(infile.read()))
+

 if __name__ == "__main__":
    cli()
--- a/tox.ini
+++ b/tox.ini
@ -33,7 +33,7 @@ commands =
  python ./test/individual_coverage.py

 [testenv:cibuild]
-passenv = TRAVIS_* AWS_* APPVEYOR_* TWINE_* RTOOL_KEY WHEEL
+passenv = TRAVIS_* APPVEYOR_* AWS_* TWINE_* DOCKER_* RTOOL_KEY WHEEL DOCKER PYINSTALLER
 deps =
  -rrequirements.txt
  pyinstaller==3.3.1
@ -41,7 +41,7 @@ deps =
  awscli
 commands =
  mitmdump --version
-  python ./release/ci.py {posargs}
+  python ./release/cibuild.py {posargs}

 [testenv:wheeltest]
 recreate = True
@ -55,7 +55,7 @@ commands =
  pathoc --version

 [testenv:docs]
-passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL
+passenv = TRAVIS_* APPVEYOR_* AWS_*
 deps =
  -rrequirements.txt
  awscli