Merge pull request #937 from mhils/single-repo

Combine mitmproxy, pathod and netlib in a single repo.

.appveyor.yml

@@ -14,19 +14,17 @@ environment:
     secure: LPjrtFrWxYhOVGXzfPRV1GjtZE/wHoKq9m/PI6hSalfysUK5p2DxTG9uHlb4Q9qV
 install:
   - "pip install --user -U pip setuptools"
-  - "pip install --user --src .. -r requirements.txt"
+  - "pip install --user -r requirements.txt"
   - "python -c \"from OpenSSL import SSL; print(SSL.SSLeay_version(SSL.SSLEAY_VERSION))\""
 test_script:
-  - "py.test -s --cov libmproxy --timeout 30"
+  - "py.test -s --cov-config .coveragerc --timeout 30 ./test/"
 cache:
   - C:\Users\appveyor\AppData\Local\pip\cache
 deploy_script:
   ps: |
     if($Env:APPVEYOR_REPO_BRANCH -match "master") {
-    git clone https://github.com/mitmproxy/release.git ..\release
+    python .\release\rtool.py bdist
-    pip install -e ..\release
+    python .\release\rtool.py upload-snapshot --bdist
-    python ..\release\rtool.py bdist
-    python ..\release\rtool.py upload-snapshot --bdist
     }
 notifications:
 - provider: Slack

.coveragerc

@@ -1,11 +1,11 @@
 [run]
 branch = True
+include = libmproxy netlib libpathod
+omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py
 
 [report]
 show_missing = True
-include = *libmproxy*
 exclude_lines =
     pragma: nocover
     pragma: no cover
     raise NotImplementedError()
-omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py

.env

@@ -1,6 +1,6 @@
 DIR="$( dirname "${BASH_SOURCE[0]}" )"
-ACTIVATE_DIR="$(if [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then echo 'bin'; else echo 'Scripts'; fi;)"
+ACTIVATE_DIR="$(if [ -f "$DIR/venv/bin/activate" ]; then echo 'bin'; else echo 'Scripts'; fi;)"
-if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/$ACTIVATE_DIR/activate" ]; then
+if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/venv/$ACTIVATE_DIR/activate" ]; then
     echo "Activating mitmproxy virtualenv..."
-    source "$DIR/../venv.mitmproxy/$ACTIVATE_DIR/activate"
+    source "$DIR/venv/$ACTIVATE_DIR/activate"
 fi

.gitattributes

@@ -1,2 +1,2 @@
-libmproxy/web/static/**/* -diff
+mitmproxy/libmproxy/web/static/**/* -diff
-web/src/js/filt/filt.js -diff
+mitmproxy/web/src/js/filt/filt.js -diff

.gitignore

@@ -1,28 +1,19 @@
 .DS_Store
 MANIFEST
-/build
+*/build
-/dist
+*/dist
-/tmp
+*/tmp
-/doc
 /venv
-/libmproxy/gui
-/release/build
 *.py[cdo]
 *.swp
 *.swo
-mitmproxy.egg-info/
+*.egg-info/
-mitmproxyc
-mitmdumpc
 .coverage
 .idea
-netlib
-pathod
-libpathod
 .cache/
 
 # UI
 
 node_modules
 bower_components
-*.compiled.js
 *.map

.landscape.yml

@@ -1,8 +1,8 @@
 ignore-paths:
-  - docs
+  - mitmproxy/docs
-  - examples
+  - mitmproxy/examples
-  - libmproxy/contrib
+  - mitmproxy/libmproxy/contrib
-  - web
+  - mitmproxy/web
 max-line-length: 140
 pylint:
   options:

.travis.yml

@@ -1,37 +1,31 @@
 sudo: false
 language: python
 
+addons:
+  apt:
+    sources:
+      # Debian sid currently holds OpenSSL 1.0.2
+      # change this with future releases!
+      - debian-sid
+    packages:
+      - libssl-dev
+
 matrix:
   fast_finish: true
   include:
     - python: 2.7
     - python: 2.7
-      env: OPENSSL=1.0.2
+      env: NO_ALPN=1
-      addons:
-        apt:
-          sources:
-            # Debian sid currently holds OpenSSL 1.0.2
-            # change this with future releases!
-            - debian-sid
-          packages:
-            - libssl-dev
-    - python: pypy
-    - python: pypy
-      env: OPENSSL=1.0.2
-      addons:
-        apt:
-          sources:
-            # Debian sid currently holds OpenSSL 1.0.2
-            # change this with future releases!
-            - debian-sid
-          packages:
-            - libssl-dev
     - language: generic
       os: osx
       osx_image: xcode7.1
+    - python: 3.5
+      env: SCOPE="netlib"
+    - python: 3.5
+      env: SCOPE="netlib" NO_ALPN=1
     - python: 2.7
       env: DOCS=1
-      script: 'cd docs && make html'
+      script: 'cd mitmproxy/docs && make html'
   allow_failures:
     - python: pypy
 
@@ -43,37 +37,23 @@ install:
       brew outdated openssl || brew upgrade openssl
       brew install python
     fi
-  - |
-      if [ "$TRAVIS_PYTHON_VERSION" = "pypy" ]; then
-        export PYENV_ROOT="$HOME/.pyenv"
-        if [ -f "$PYENV_ROOT/bin/pyenv" ]; then
-          pushd "$PYENV_ROOT" && git pull && popd
-        else
-          rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
-        fi
-        export PYPY_VERSION="4.0.1"
-        "$PYENV_ROOT/bin/pyenv" install --skip-existing "pypy-$PYPY_VERSION"
-        virtualenv --python="$PYENV_ROOT/versions/pypy-$PYPY_VERSION/bin/python" "$HOME/virtualenvs/pypy-$PYPY_VERSION"
-        source "$HOME/virtualenvs/pypy-$PYPY_VERSION/bin/activate"
-      fi
   - "pip install -U pip setuptools"
-  - "pip install --src .. -r requirements.txt"
+  - "pip install -r requirements.txt"
 
 before_script:
   - "openssl version -a"
 
 script:
-  - "py.test -s --cov libmproxy --timeout 30"
+  - "py.test -s --cov-config .coveragerc --timeout 30 ./test/$SCOPE"
 
 after_success:
   - coveralls
   - |
     if [[ $TRAVIS_OS_NAME == "osx" && $TRAVIS_BRANCH == "master" && $TRAVIS_PULL_REQUEST == "false" ]]
     then
-        git clone -q https://github.com/mitmproxy/release.git ../release
+        pip install -e ./release
-        pip install -e ../release
+        python ./release/rtool.py bdist
-        python ../release/rtool.py bdist
+        python ./release/rtool.py upload-snapshot --sdist --bdist --wheel
-        python ../release/rtool.py upload-snapshot --sdist --bdist --wheel
     fi
 
 notifications:

README.rst

@@ -1,45 +1,32 @@
-|travis| |coveralls| |downloads| |latest-release| |python-versions|
+mitmproxy
+^^^^^^^^^
 
-``mitmproxy`` is an interactive, SSL-capable man-in-the-middle proxy for HTTP
+|travis| |coveralls| |downloads| |latest_release| |python_versions|
-with a console interface.
+
+This repository contains the **mitmproxy** and **pathod** projects, as well as their shared networking library, **netlib**.
+
+``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console interface.
 
 ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP.
 
-``libmproxy`` is the library that mitmproxy and mitmdump are built on.
+``pathoc`` and ``pathod`` are perverse HTTP client and server applications designed to let you craft almost any conceivable HTTP request, including ones that creatively violate the standards.
+
 
 Documentation & Help
 --------------------
 
-Documentation, tutorials and distribution packages can be found on the
+Documentation, tutorials and precompiled binaries can be found on the mitmproxy and pathod websites.
-mitmproxy website.
 
-|site|
+|mitmproxy_site| |pathod_site|
 
-Installation Instructions are available in the docs.
+The latest documentation for mitmproxy is also available on ReadTheDocs.
 
-|docs|
+|mitmproxy_docs|
 
 You can join our developer chat on Slack.
 
 |slack|
 
-Features
---------
-
-- Intercept HTTP requests and responses and modify them on the fly.
-- Save complete HTTP conversations for later replay and analysis.
-- Replay the client-side of an HTTP conversations.
-- Replay HTTP responses of a previously recorded server.
-- Reverse proxy mode to forward traffic to a specified server.
-- Transparent proxy mode on OSX and Linux.
-- Make scripted changes to HTTP traffic using Python.
-- SSL certificates for interception are generated on the fly.
-- And much, much more.
-
-``mitmproxy`` is tested and developed on OSX, Linux and OpenBSD.
-On Windows, only mitmdump is supported, which does not have a graphical user interface.
-
-
 
 Hacking
 -------
@@ -51,28 +38,25 @@ Then do the following:
 .. code-block:: text
 
     git clone https://github.com/mitmproxy/mitmproxy.git
-    git clone https://github.com/mitmproxy/netlib.git
-    git clone https://github.com/mitmproxy/pathod.git
     cd mitmproxy
     ./dev
 
 
-The *dev* script will create a virtualenv environment in a directory called
+The *dev* script will create a virtualenv environment in a directory called "venv",
-"venv.mitmproxy", and install all of mitmproxy's development requirements, plus
+and install all mandatory and optional dependencies into it.
-all optional modules. The primary mitmproxy components - mitmproxy, netlib and
+The primary mitmproxy components - mitmproxy, netlib and pathod - are installed as "editable",
-pathod - are all installed "editable", so any changes to the source in the git
+so any changes to the source in the repository will be reflected live in the virtualenv.
-checkouts will be reflected live in the virtualenv.
 
 To confirm that you're up and running, activate the virtualenv, and run the
 mitmproxy test suite:
 
 .. code-block:: text
 
-    . ../venv.mitmproxy/bin/activate # ..\venv.mitmproxy\Scripts\activate.bat on Windows
+    . venv/bin/activate # venv\Scripts\activate.bat on Windows
-    py.test -n 4 --cov libmproxy
+    py.test --cov-config .coveragerc test
 
-Note that the main executables for the project - ``mitmdump``, ``mitmproxy`` and
+Note that the main executables for the project - ``mitmdump``, ``mitmproxy``,
-``mitmweb`` - are all created within the virtualenv. After activating the
+``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the virtualenv. After activating the
 virtualenv, they will be on your $PATH, and you can run them like any other
 command:
 
@@ -92,10 +76,10 @@ requirements installed, and you can simply run the test suite:
 
 .. code-block:: text
 
-    py.test -n 4 --cov libmproxy
+    py.test --cov-config .coveragerc test
 
 Please ensure that all patches are accompanied by matching changes in the test
-suite. The project maintains 100% test coverage.
+suite. The project tries to maintain 100% test coverage.
 
 
 Docs
@@ -116,35 +100,39 @@ The last command invokes `sphinx-autobuild`_, which watches the Sphinx directory
 the documentation when a change is detected.
 
 
-.. |site| image:: https://img.shields.io/badge/https%3A%2F%2F-mitmproxy.org-blue.svg
+.. |mitmproxy_site| image:: https://shields.mitmproxy.org/badge/https%3A%2F%2F-mitmproxy.org-blue.svg
     :target: https://mitmproxy.org/
     :alt: mitmproxy.org
 
-.. |docs| image:: https://readthedocs.org/projects/mitmproxy/badge/
+.. |pathod_site| image:: https://shields.mitmproxy.org/badge/https%3A%2F%2F-pathod.net-blue.svg
+    :target: https://pathod.net/
+    :alt: pathod.net
+
+.. |mitmproxy_docs| image:: https://readthedocs.org/projects/mitmproxy/badge/
     :target: http://docs.mitmproxy.org/en/latest/
-    :alt: Documentation
+    :alt: mitmproxy documentation
 
 .. |slack| image:: http://slack.mitmproxy.org/badge.svg
     :target: http://slack.mitmproxy.org/
     :alt: Slack Developer Chat
 
-.. |travis| image:: https://img.shields.io/travis/mitmproxy/mitmproxy/master.svg
+.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg
     :target: https://travis-ci.org/mitmproxy/mitmproxy
     :alt: Build Status
 
-.. |coveralls| image:: https://img.shields.io/coveralls/mitmproxy/mitmproxy/master.svg
+.. |coveralls| image:: https://shields.mitmproxy.org/coveralls/mitmproxy/mitmproxy/master.svg
     :target: https://coveralls.io/r/mitmproxy/mitmproxy
     :alt: Coverage Status
 
-.. |downloads| image:: https://img.shields.io/pypi/dm/mitmproxy.svg?color=orange
+.. |downloads| image:: https://shields.mitmproxy.org/pypi/dm/mitmproxy.svg?color=orange
     :target: https://pypi.python.org/pypi/mitmproxy
     :alt: Downloads
 
-.. |latest-release| image:: https://img.shields.io/pypi/v/mitmproxy.svg
+.. |latest_release| image:: https://shields.mitmproxy.org/pypi/v/mitmproxy.svg
     :target: https://pypi.python.org/pypi/mitmproxy
     :alt: Latest Version
 
-.. |python-versions| image:: https://img.shields.io/pypi/pyversions/mitmproxy.svg
+.. |python_versions| image:: https://shields.mitmproxy.org/pypi/pyversions/mitmproxy.svg
     :target: https://pypi.python.org/pypi/mitmproxy
     :alt: Supported Python versions
 

dev.bat

@@ -1,11 +1,11 @@
 @echo off
-set VENV=..\venv.mitmproxy
+set VENV=.\venv
 
 virtualenv %VENV% --always-copy
 if %errorlevel% neq 0 exit /b %errorlevel%
 call %VENV%\Scripts\activate.bat
 if %errorlevel% neq 0 exit /b %errorlevel%
-pip install --src .. -r requirements.txt
+pip install -r requirements.txt
 if %errorlevel% neq 0 exit /b %errorlevel%
 
 echo.

dev.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 set -e
-VENV=../venv.mitmproxy
+VENV=./venv
 
 python -m virtualenv $VENV --always-copy
 . $VENV/bin/activate
-pip install --src .. -r requirements.txt
+pip install -r requirements.txt
 
 echo ""
 echo "* Created virtualenv environment in $VENV."

mitmproxy/README.rst

@@ -0,0 +1,73 @@
+|travis| |coveralls| |downloads| |latest_release| |python_versions|
+
+``mitmproxy`` is an interactive, SSL/TLS-capable man-in-the-middle proxy for HTTP
+with a console interface.
+
+``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP.
+
+
+Features
+--------
+
+- Intercept HTTP requests and responses and modify them on the fly.
+- Save complete HTTP conversations for later replay and analysis.
+- Replay the client-side of an HTTP conversations.
+- Replay HTTP responses of a previously recorded server.
+- Reverse proxy mode to forward traffic to a specified server.
+- Transparent proxy mode on OSX and Linux.
+- Make scripted changes to HTTP traffic using Python.
+- SSL/TLS certificates for interception are generated on the fly.
+- And much, much more.
+
+``mitmproxy`` is tested and developed on Mac OSX and Linux.
+On Windows, only mitmdump is supported, which does not have a graphical user interface.
+
+
+Documentation & Help
+--------------------
+
+Documentation, tutorials and distribution packages can be found on the
+mitmproxy website.
+
+|mitmproxy_site|
+
+Installation Instructions are available in the documentation.
+
+|mitmproxy_docs|
+
+You can join our developer chat on Slack.
+
+|slack|
+
+
+.. |mitmproxy_site| image:: https://shields.mitmproxy.org/badge/https%3A%2F%2F-mitmproxy.org-blue.svg
+    :target: https://mitmproxy.org/
+    :alt: mitmproxy.org
+
+.. |mitmproxy_docs| image:: https://readthedocs.org/projects/mitmproxy/badge/
+    :target: http://docs.mitmproxy.org/en/latest/
+    :alt: mitmproxy documentation
+
+.. |slack| image:: http://slack.mitmproxy.org/badge.svg
+    :target: http://slack.mitmproxy.org/
+    :alt: Slack Developer Chat
+
+.. |travis| image:: https://shields.mitmproxy.org/travis/mitmproxy/mitmproxy/master.svg
+    :target: https://travis-ci.org/mitmproxy/mitmproxy
+    :alt: Build Status
+
+.. |coveralls| image:: https://shields.mitmproxy.org/coveralls/mitmproxy/mitmproxy/master.svg
+    :target: https://coveralls.io/r/mitmproxy/mitmproxy
+    :alt: Coverage Status
+
+.. |downloads| image:: https://shields.mitmproxy.org/pypi/dm/mitmproxy.svg?color=orange
+    :target: https://pypi.python.org/pypi/mitmproxy
+    :alt: Downloads
+
+.. |latest_release| image:: https://shields.mitmproxy.org/pypi/v/mitmproxy.svg
+    :target: https://pypi.python.org/pypi/mitmproxy
+    :alt: Latest Version
+
+.. |python_versions| image:: https://shields.mitmproxy.org/pypi/pyversions/mitmproxy.svg
+    :target: https://pypi.python.org/pypi/mitmproxy
+    :alt: Supported Python versions