Merge branch 'master' of github.com:mitmproxy/mitmproxy into list

Conflicts:
	mitmproxy/web/static/app.js
	web/src/js/app.jsx
	web/src/js/connection.js
	web/src/js/ducks/websocket.js

docs/install.rst

@@ -11,8 +11,10 @@ Installation On Ubuntu
 Ubuntu comes with Python but we need to install pip, python-dev and several libraries.
 This was tested on a fully patched installation of Ubuntu 14.04.
 
->>> sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev
+.. code:: bash
->>> sudo pip install mitmproxy
+
+   sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev
+   sudo pip install mitmproxy  # or pip install --user mitmproxy
 
 Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
@@ -27,6 +29,20 @@ get set up to contribute to the project, install the dependencies as you would f
 mitmproxy installation (see :ref:`install-ubuntu`).
 Then see the Hacking_ section of the README on GitHub.
 
+.. _install-fedora:
+
+Installation On Fedora
+----------------------
+
+Fedora comes with Python but we need to install pip, python-dev and several libraries.
+This was tested on a fully patched installation of Fedora 23.
+
+.. code:: bash
+
+   sudo dnf install -y python-pip python-devel libffi-devel openssl-devel libxml2-devel libxslt-devel libpng-devel libjpeg-devel
+   sudo pip install mitmproxy  # or pip install --user mitmproxy
+
+Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal.
 
 
 .. _install-arch:
@@ -87,7 +103,9 @@ Installation On Windows
 
 First, install the latest version of Python 2.7 from the `Python website`_.
 If you already have an older version of Python 2.7 installed, make sure to install pip_
-(pip is included in Python 2.7.9+ by default).
+(pip is included in Python 2.7.9+ by default). If pip aborts with an error, make sure you are using the current version of pip.
+
+>>> python -m pip install --upgrade pip
 
 Next, add Python and the Python Scripts directory to your **PATH** variable.
 You can do this easily by running the following in powershell:

mitmproxy/platform/pf.py

@@ -8,6 +8,7 @@ def lookup(address, port, s):
 
         Returns an (address, port) tuple, or None.
     """
+    s = s.decode()
     spec = "%s:%s" % (address, port)
     for i in s.split("\n"):
         if "ESTABLISHED:ESTABLISHED" in i and spec in i:

mitmproxy/proxy/config.py

@@ -199,7 +199,7 @@ def process_proxy_options(parser, options):
                 password_manager = authentication.PassManHtpasswd(
                     options.auth_htpasswd)
             except ValueError as v:
-                return parser.error(v.message)
+                return parser.error(v)
         authenticator = authentication.BasicProxyAuth(password_manager, "mitmproxy")
     else:
         authenticator = authentication.NullProxyAuth(None)

netlib/http/http2/framereader.py

@@ -1,6 +1,7 @@
 import codecs
 
 import hyperframe
+from ...exceptions import HttpException
 
 
 def http2_read_raw_frame(rfile):
@@ -8,7 +9,7 @@ def http2_read_raw_frame(rfile):
     length = int(codecs.encode(header[:3], 'hex_codec'), 16)
 
     if length == 4740180:
-        raise ValueError("Length field looks more like HTTP/1.1: %s" % rfile.peek(20))
+        raise HttpException("Length field looks more like HTTP/1.1:\n{}".format(rfile.read(-1)))
 
     body = rfile.safe_read(length)
     return [header, body]

test/mitmproxy/test_protocol_http1.py

@@ -18,14 +18,14 @@ class TestInvalidRequests(tservers.HTTPProxyTest):
         p = self.pathoc()
         r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port))
         assert r.status_code == 400
-        assert "Invalid HTTP request form" in r.content
+        assert b"Invalid HTTP request form" in r.content
 
     def test_relative_request(self):
         p = self.pathoc_raw()
         p.connect()
         r = p.request("get:/p/200")
         assert r.status_code == 400
-        assert "Invalid HTTP request form" in r.content
+        assert b"Invalid HTTP request form" in r.content
 
 
 class TestExpectHeader(tservers.HTTPProxyTest):
@@ -43,8 +43,8 @@ class TestExpectHeader(tservers.HTTPProxyTest):
         )
         client.wfile.flush()
 
-        assert client.rfile.readline() == "HTTP/1.1 100 Continue\r\n"
+        assert client.rfile.readline() == b"HTTP/1.1 100 Continue\r\n"
-        assert client.rfile.readline() == "\r\n"
+        assert client.rfile.readline() == b"\r\n"
 
         client.wfile.write(b"0123456789abcdef\r\n")
         client.wfile.flush()

test/mitmproxy/test_protocol_http2.py

@@ -13,6 +13,7 @@ from mitmproxy.cmdline import APP_HOST, APP_PORT
 
 import netlib
 from ..netlib import tservers as netlib_tservers
+from netlib.exceptions import HttpException
 from netlib.http.http2 import framereader
 
 from . import tservers
@@ -50,6 +51,9 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase):
                 try:
                     raw = b''.join(framereader.http2_read_raw_frame(self.rfile))
                     events = h2_conn.receive_data(raw)
+                except HttpException:
+                    print(traceback.format_exc())
+                    assert False
                 except:
                     break
                 self.wfile.write(h2_conn.data_to_send())
@@ -60,9 +64,7 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase):
                         if not self.server.handle_server_event(event, h2_conn, self.rfile, self.wfile):
                             done = True
                             break
-                    except Exception as e:
+                    except:
-                        print(repr(e))
-                        print(traceback.format_exc())
                         done = True
                         break
 
@@ -200,9 +202,12 @@ class TestSimple(_Http2TestBase, _Http2ServerBase):
         done = False
         while not done:
             try:
-                events = h2_conn.receive_data(b''.join(framereader.http2_read_raw_frame(client.rfile)))
+                raw = b''.join(framereader.http2_read_raw_frame(client.rfile))
-            except:
+                events = h2_conn.receive_data(raw)
-                break
+            except HttpException:
+                print(traceback.format_exc())
+                assert False
+
             client.wfile.write(h2_conn.data_to_send())
             client.wfile.flush()
 
@@ -270,9 +275,12 @@ class TestWithBodies(_Http2TestBase, _Http2ServerBase):
         done = False
         while not done:
             try:
-                events = h2_conn.receive_data(b''.join(framereader.http2_read_raw_frame(client.rfile)))
+                raw = b''.join(framereader.http2_read_raw_frame(client.rfile))
-            except:
+                events = h2_conn.receive_data(raw)
-                break
+            except HttpException:
+                print(traceback.format_exc())
+                assert False
+
             client.wfile.write(h2_conn.data_to_send())
             client.wfile.flush()
 
@@ -364,6 +372,9 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase):
             try:
                 raw = b''.join(framereader.http2_read_raw_frame(client.rfile))
                 events = h2_conn.receive_data(raw)
+            except HttpException:
+                print(traceback.format_exc())
+                assert False
             except:
                 break
             client.wfile.write(h2_conn.data_to_send())
@@ -412,9 +423,12 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase):
         responses = 0
         while not done:
             try:
-                events = h2_conn.receive_data(b''.join(framereader.http2_read_raw_frame(client.rfile)))
+                raw = b''.join(framereader.http2_read_raw_frame(client.rfile))
-            except:
+                events = h2_conn.receive_data(raw)
-                break
+            except HttpException:
+                print(traceback.format_exc())
+                assert False
+
             client.wfile.write(h2_conn.data_to_send())
             client.wfile.flush()
 
@@ -481,6 +495,9 @@ class TestConnectionLost(_Http2TestBase, _Http2ServerBase):
             try:
                 raw = b''.join(framereader.http2_read_raw_frame(client.rfile))
                 h2_conn.receive_data(raw)
+            except HttpException:
+                print(traceback.format_exc())
+                assert False
             except:
                 break
             try:

tox.ini

@@ -7,7 +7,7 @@ deps =
   codecov>=2.0.5
 passenv = CI TRAVIS_BUILD_ID TRAVIS TRAVIS_BRANCH TRAVIS_JOB_NUMBER TRAVIS_PULL_REQUEST TRAVIS_JOB_ID TRAVIS_REPO_SLUG TRAVIS_COMMIT
 setenv =
-    PY3TESTS =  test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py
+    PY3TESTS =  test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py
 
 [testenv:py27]
 commands =

web/src/js/app.jsx

@@ -1,16 +1,15 @@
-import React from "react"
+import React from 'react'
 import { render } from 'react-dom'
 import { applyMiddleware, createStore } from 'redux'
 import { Provider } from 'react-redux'
 import createLogger from 'redux-logger'
 import thunkMiddleware from 'redux-thunk'
-import { Route, Router as ReactRouter, hashHistory, Redirect } from "react-router"
+import { Route, Router as ReactRouter, hashHistory, Redirect } from 'react-router'
 
-import Connection from "./connection"
+import ProxyApp from './components/ProxyApp'
-import ProxyApp from "./components/ProxyApp"
 import MainView from './components/MainView'
 import rootReducer from './ducks/index'
-import { add as addLog } from "./ducks/eventLog"
+import { add as addLog } from './ducks/eventLog'
 
 // logger must be last
 const store = createStore(
@@ -18,14 +17,13 @@ const store = createStore(
     applyMiddleware(thunkMiddleware, createLogger())
 )
 
+// @todo move to ProxyApp
 window.addEventListener('error', msg => {
     store.dispatch(addLog(msg))
 })
 
 // @todo remove this
 document.addEventListener('DOMContentLoaded', () => {
-    window.ws = new Connection("/updates", store.dispatch)
-
     render(
         <Provider store={store}>
             <ReactRouter history={hashHistory}>

web/src/js/components/ProxyApp.jsx

@@ -3,6 +3,7 @@ import ReactDOM from 'react-dom'
 import _ from 'lodash'
 import { connect } from 'react-redux'
 
+import { init as appInit, destruct as appDestruct } from '../ducks/app'
 import Header from './Header'
 import EventLog from './EventLog'
 import Footer from './Footer'
@@ -26,27 +27,8 @@ class ProxyAppMain extends Component {
         this.updateLocation = this.updateLocation.bind(this)
     }
 
-    /**
+    componentWillMount() {
-     * @todo move to actions
+        this.props.appInit()
-     */
-    updateLocation(pathname, queryUpdate) {
-        if (pathname === undefined) {
-            pathname = this.props.location.pathname
-        }
-        const query = this.props.location.query
-        for (const key of Object.keys(queryUpdate || {})) {
-            query[key] = queryUpdate[key] || undefined
-        }
-        this.context.router.replace({ pathname, query })
-    }
-
-    /**
-     * @todo pass in with props
-     */
-    getQuery() {
-        // For whatever reason, react-router always returns the same object, which makes comparing
-        // the current props with nextProps impossible. As a workaround, we just clone the query object.
-        return _.clone(this.props.location.query)
     }
 
     /**
@@ -56,6 +38,10 @@ class ProxyAppMain extends Component {
         this.focus()
     }
 
+    componentWillUnmount() {
+        this.props.appDestruct()
+    }
+
     /**
      * @todo use props
      */
@@ -110,6 +96,29 @@ class ProxyAppMain extends Component {
         e.preventDefault()
     }
 
+    /**
+     * @todo move to actions
+     */
+    updateLocation(pathname, queryUpdate) {
+        if (pathname === undefined) {
+            pathname = this.props.location.pathname
+        }
+        const query = this.props.location.query
+        for (const key of Object.keys(queryUpdate || {})) {
+            query[key] = queryUpdate[key] || undefined
+        }
+        this.context.router.replace({ pathname, query })
+    }
+
+    /**
+     * @todo pass in with props
+     */
+    getQuery() {
+        // For whatever reason, react-router always returns the same object, which makes comparing
+        // the current props with nextProps impossible. As a workaround, we just clone the query object.
+        return _.clone(this.props.location.query)
+    }
+
     render() {
         const { showEventLog, location, children } = this.props
         const query = this.getQuery()
@@ -132,5 +141,10 @@ class ProxyAppMain extends Component {
 export default connect(
     state => ({
         showEventLog: state.eventLog.visible,
-    })
+        settings: state.settings.settings,
+    }),
+    {
+        appInit,
+        appDestruct,
+    }
 )(ProxyAppMain)

web/src/js/connection.js

@@ -1,49 +0,0 @@
-import {ConnectionActions} from "./actions.js";
-import {AppDispatcher} from "./dispatcher.js";
-import * as webSocketActions from "./ducks/websocket"
-import * as eventLogActions from "./ducks/eventLog"
-import * as flowActions from "./ducks/flows"
-import * as settingsActions from './ducks/settings'
-
-export default function Connection(url, dispatch) {
-    if (url[0] === "/") {
-        url = location.origin.replace("http", "ws") + url;
-    }
-
-    var ws = new WebSocket(url);
-    ws.onopen = function () {
-        dispatch(webSocketActions.connected())
-        dispatch(settingsActions.fetchSettings())
-        dispatch(eventLogActions.fetchData())
-        dispatch(flowActions.fetchData())
-        // workaround to make sure that our state is already available.
-            .then(() => {
-            console.log("flows are loaded now")
-            ConnectionActions.open()
-        })
-    };
-    ws.onmessage = function (m) {
-        var message = JSON.parse(m.data);
-        AppDispatcher.dispatchServerAction(message);
-        switch (message.type) {
-            case eventLogActions.WS_MSG_TYPE:
-                return dispatch(eventLogActions.handleWsMsg(message))
-            case flowActions.WS_MSG_TYPE:
-                return dispatch(flowActions.handleWsMsg(message))
-            case settingsActions.UPDATE_SETTINGS:
-                return dispatch(settingsActions.handleWsMsg(message))
-            default:
-                console.warn("unknown message", message)
-        }
-    };
-    ws.onerror = function () {
-        ConnectionActions.error();
-        dispatch(eventLogActions.add("WebSocket connection error."));
-    };
-    ws.onclose = function () {
-        ConnectionActions.close();
-        dispatch(eventLogActions.add("WebSocket connection closed."));
-        dispatch(webSocketActions.disconnected());
-    };
-    return ws;
-}

web/src/js/ducks/app.js

@@ -0,0 +1,27 @@
+import { connect as wsConnect, disconnect as wsDisconnect } from './websocket'
+
+export const INIT = 'APP_INIT'
+
+const defaultState = {}
+
+export function reduce(state = defaultState, action) {
+    switch (action.type) {
+
+        default:
+            return state
+    }
+}
+
+export function init() {
+    return dispatch => {
+        dispatch(wsConnect())
+        dispatch({ type: INIT })
+    }
+}
+
+export function destruct() {
+    return dispatch => {
+        dispatch(wsDisconnect())
+        dispatch({ type: DESTRUCT })
+    }
+}

web/src/js/ducks/websocket.js

@@ -1,34 +1,118 @@
-const CONNECTED = 'WEBSOCKET_CONNECTED'
+import { ConnectionActions } from '../actions.js'
-const DISCONNECTED = 'WEBSOCKET_DISCONNECTED'
+import { AppDispatcher } from '../dispatcher.js'
+import * as eventLogActions from './eventLog'
+import * as flowsActions from './flows'
+import * as settingsActions from './settings'
 
 export const CMD_ADD = 'add'
 export const CMD_UPDATE = 'update'
 export const CMD_REMOVE = 'remove'
 export const CMD_RESET = 'reset'
 
-const defaultState = {
+export const SYM_SOCKET = Symbol('WEBSOCKET_SYM_SOCKET')
-    connected: false,
+
+export const CONNECT = 'WEBSOCKET_CONNECT'
+export const CONNECTED = 'WEBSOCKET_CONNECTED'
+export const DISCONNECT = 'WEBSOCKET_DISCONNECT'
+export const DISCONNECTED = 'WEBSOCKET_DISCONNECTED'
+export const ERROR = 'WEBSOCKET_ERROR'
+export const MESSAGE = 'WEBSOCKET_MESSAGE'
+
 /* we may want to have an error message attribute here at some point */
-}
+const defaultState = { connected: false, socket: null }
-export default function reducer(state = defaultState, action) {
+
+export default function reduce(state = defaultState, action) {
     switch (action.type) {
+
+        case CONNECT:
+            return { ...state, [SYM_SOCKET]: action.socket }
+
         case CONNECTED:
-            return {
+            return { ...state, connected: true }
-                connected: true
+
-            }
+        case DISCONNECT:
+            return { ...state, connected: false }
+
         case DISCONNECTED:
-            return {
+            return { ...state, [SYM_SOCKET]: null, connected: false }
-                connected: false
+
-            }
         default:
             return state
     }
 }
 
+export function connect() {
+    return dispatch => {
+        const socket = new WebSocket(location.origin.replace('http', 'ws') + '/updates')
 
-export function connected() {
+        // @todo remove this
-    return {type: CONNECTED}
+        window.ws = socket
+
+        socket.addEventListener('open', () => dispatch(onConnect()))
+        socket.addEventListener('close', () => dispatch(onDisconnect()))
+        socket.addEventListener('message', msg => dispatch(onMessage(msg)))
+        socket.addEventListener('error', error => dispatch(onError(error)))
+
+        dispatch({ type: CONNECT, socket })
+
+        return socket
+    }
+}
+
+export function disconnect() {
+    return (dispatch, getState) => {
+        getState().settings[SYM_SOCKET].close()
+        dispatch({ type: DISCONNECT })
+    }
+}
+
+export function onConnect() {
+    // workaround to make sure that our state is already available.
+    return dispatch => {
+        dispatch({ type: CONNECTED })
+        dispatch(settingsActions.fetchSettings())
+        dispatch(flowsActions.fetchFlows()).then(() => ConnectionActions.open())
+    }
+}
+
+export function onMessage(msg) {
+    return dispatch => {
+        const data = JSON.parse(msg.data)
+
+        AppDispatcher.dispatchServerAction(data)
+
+        switch (data.type) {
+
+            case eventLogActions.WS_MSG_TYPE:
+                return dispatch(eventLogActions.handleWsMsg(data))
+
+            case flowsActions.WS_MSG_TYPE:
+                return dispatch(flowsActions.handleWsMsg(data))
+
+            case settingsActions.UPDATE_SETTINGS:
+                return dispatch(settingsActions.handleWsMsg(data))
+
+            default:
+                console.warn('unknown message', data)
+        }
+
+        dispatch({ type: MESSAGE, msg })
+    }
+}
+
+export function onDisconnect() {
+    return dispatch => {
+        ConnectionActions.close()
+        dispatch(eventLogActions.addLogEntry('WebSocket connection closed.'))
+        dispatch({ type: DISCONNECTED })
+    }
+}
+
+export function onError(error) {
+    // @todo let event log subscribe WebSocketActions.ERROR
+    return dispatch => {
+        ConnectionActions.error()
+        dispatch(eventLogActions.addLogEntry('WebSocket connection error.'))
+        dispatch({ type: ERROR, error })
     }
-export function disconnected() {
-    return {type: DISCONNECTED}
 }