Merge remote-tracking branch 'upstream/master'

.travis.yml

@@ -22,9 +22,9 @@ matrix:
       git:
         depth: 9999999
     - python: 3.5
-      env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py"
+      env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py test/pathod/test_language_http.py"
     - python: 3.5
-      env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py" NO_ALPN=1
+      env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py test/pathod/test_language_http.py" NO_ALPN=1
     - python: 2.7
       env: DOCS=1
       script: 'cd docs && make html'

mitmproxy/web/__init__.py

@@ -1,2 +1,2 @@
 from mitmproxy.web import master
-__all__ = ["master"]
+__all__ = ["master"]

mitmproxy/web/static/app.js

@@ -1212,251 +1212,221 @@ var _utils2 = require("../utils.js");
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
-var TLSColumn = _react2.default.createClass({
-    displayName: "TLSColumn",
+function _objectWithoutProperties(obj, keys) { var target = {}; for (var i in obj) { if (keys.indexOf(i) >= 0) continue; if (!Object.prototype.hasOwnProperty.call(obj, i)) continue; target[i] = obj[i]; } return target; }
 
-    statics: {
-        Title: _react2.default.createClass({
-            displayName: "Title",
+function TLSColumn(_ref) {
+    var flow = _ref.flow;
 
-            render: function render() {
-                return _react2.default.createElement("th", _extends({}, this.props, { className: "col-tls " + (this.props.className || "") }));
-            }
-        }),
-        sortKeyFun: function sortKeyFun(flow) {
-            return flow.request.scheme;
-        }
-    },
-    render: function render() {
-        var flow = this.props.flow;
-        var ssl = flow.request.scheme === "https";
-        var classes;
-        if (ssl) {
-            classes = "col-tls col-tls-https";
-        } else {
-            classes = "col-tls col-tls-http";
-        }
-        return _react2.default.createElement("td", { className: classes });
+    var ssl = flow.request.scheme === "https";
+    var classes = void 0;
+    if (ssl) {
+        classes = "col-tls col-tls-https";
+    } else {
+        classes = "col-tls col-tls-http";
     }
-});
+    return _react2.default.createElement("td", { className: classes });
+}
+TLSColumn.Title = function (_ref2) {
+    var _ref2$className = _ref2.className;
+    var className = _ref2$className === undefined ? "" : _ref2$className;
 
-var IconColumn = _react2.default.createClass({
-    displayName: "IconColumn",
+    var props = _objectWithoutProperties(_ref2, ["className"]);
 
-    statics: {
-        Title: _react2.default.createClass({
-            displayName: "Title",
+    return _react2.default.createElement("th", _extends({}, props, { className: "col-tls " + className }));
+};
+TLSColumn.sortKeyFun = function (flow) {
+    return flow.request.scheme;
+};
 
-            render: function render() {
-                return _react2.default.createElement("th", _extends({}, this.props, { className: "col-icon " + (this.props.className || "") }));
-            }
-        })
-    },
-    render: function render() {
-        var flow = this.props.flow;
+function IconColumn(_ref3) {
+    var flow = _ref3.flow;
 
-        var icon;
-        if (flow.response) {
-            var contentType = _utils.ResponseUtils.getContentType(flow.response);
+    var icon = void 0;
+    if (flow.response) {
+        var contentType = _utils.ResponseUtils.getContentType(flow.response);
 
-            //TODO: We should assign a type to the flow somewhere else.
-            if (flow.response.status_code === 304) {
-                icon = "resource-icon-not-modified";
-            } else if (300 <= flow.response.status_code && flow.response.status_code < 400) {
-                icon = "resource-icon-redirect";
-            } else if (contentType && contentType.indexOf("image") >= 0) {
-                icon = "resource-icon-image";
-            } else if (contentType && contentType.indexOf("javascript") >= 0) {
-                icon = "resource-icon-js";
-            } else if (contentType && contentType.indexOf("css") >= 0) {
-                icon = "resource-icon-css";
-            } else if (contentType && contentType.indexOf("html") >= 0) {
-                icon = "resource-icon-document";
-            }
+        //TODO: We should assign a type to the flow somewhere else.
+        if (flow.response.status_code === 304) {
+            icon = "resource-icon-not-modified";
+        } else if (300 <= flow.response.status_code && flow.response.status_code < 400) {
+            icon = "resource-icon-redirect";
+        } else if (contentType && contentType.indexOf("image") >= 0) {
+            icon = "resource-icon-image";
+        } else if (contentType && contentType.indexOf("javascript") >= 0) {
+            icon = "resource-icon-js";
+        } else if (contentType && contentType.indexOf("css") >= 0) {
+            icon = "resource-icon-css";
+        } else if (contentType && contentType.indexOf("html") >= 0) {
+            icon = "resource-icon-document";
         }
-        if (!icon) {
-            icon = "resource-icon-plain";
-        }
-
-        icon += " resource-icon";
-        return _react2.default.createElement(
-            "td",
-            { className: "col-icon" },
-            _react2.default.createElement("div", { className: icon })
-        );
     }
-});
-
-var PathColumn = _react2.default.createClass({
-    displayName: "PathColumn",
-
-    statics: {
-        Title: _react2.default.createClass({
-            displayName: "Title",
-
-            render: function render() {
-                return _react2.default.createElement(
-                    "th",
-                    _extends({}, this.props, { className: "col-path " + (this.props.className || "") }),
-                    "Path"
-                );
-            }
-        }),
-        sortKeyFun: function sortKeyFun(flow) {
-            return _utils.RequestUtils.pretty_url(flow.request);
-        }
-    },
-    render: function render() {
-        var flow = this.props.flow;
-        return _react2.default.createElement(
-            "td",
-            { className: "col-path" },
-            flow.request.is_replay ? _react2.default.createElement("i", { className: "fa fa-fw fa-repeat pull-right" }) : null,
-            flow.intercepted ? _react2.default.createElement("i", { className: "fa fa-fw fa-pause pull-right" }) : null,
-            _utils.RequestUtils.pretty_url(flow.request)
-        );
+    if (!icon) {
+        icon = "resource-icon-plain";
     }
-});
 
-var MethodColumn = _react2.default.createClass({
-    displayName: "MethodColumn",
+    icon += " resource-icon";
+    return _react2.default.createElement(
+        "td",
+        { className: "col-icon" },
+        _react2.default.createElement("div", { className: icon })
+    );
+}
+IconColumn.Title = function (_ref4) {
+    var _ref4$className = _ref4.className;
+    var className = _ref4$className === undefined ? "" : _ref4$className;
 
-    statics: {
-        Title: _react2.default.createClass({
-            displayName: "Title",
+    var props = _objectWithoutProperties(_ref4, ["className"]);
 
-            render: function render() {
-                return _react2.default.createElement(
-                    "th",
-                    _extends({}, this.props, { className: "col-method " + (this.props.className || "") }),
-                    "Method"
-                );
-            }
-        }),
-        sortKeyFun: function sortKeyFun(flow) {
-            return flow.request.method;
-        }
-    },
-    render: function render() {
-        var flow = this.props.flow;
-        return _react2.default.createElement(
-            "td",
-            { className: "col-method" },
-            flow.request.method
-        );
+    return _react2.default.createElement("th", _extends({}, props, { className: "col-icon " + className }));
+};
+
+function PathColumn(_ref5) {
+    var flow = _ref5.flow;
+
+    return _react2.default.createElement(
+        "td",
+        { className: "col-path" },
+        flow.request.is_replay ? _react2.default.createElement("i", { className: "fa fa-fw fa-repeat pull-right" }) : null,
+        flow.intercepted ? _react2.default.createElement("i", { className: "fa fa-fw fa-pause pull-right" }) : null,
+        _utils.RequestUtils.pretty_url(flow.request)
+    );
+}
+PathColumn.Title = function (_ref6) {
+    var _ref6$className = _ref6.className;
+    var className = _ref6$className === undefined ? "" : _ref6$className;
+
+    var props = _objectWithoutProperties(_ref6, ["className"]);
+
+    return _react2.default.createElement(
+        "th",
+        _extends({}, props, { className: "col-path " + className }),
+        "Path"
+    );
+};
+PathColumn.sortKeyFun = function (flow) {
+    return _utils.RequestUtils.pretty_url(flow.request);
+};
+
+function MethodColumn(_ref7) {
+    var flow = _ref7.flow;
+
+    return _react2.default.createElement(
+        "td",
+        { className: "col-method" },
+        flow.request.method
+    );
+}
+MethodColumn.Title = function (_ref8) {
+    var _ref8$className = _ref8.className;
+    var className = _ref8$className === undefined ? "" : _ref8$className;
+
+    var props = _objectWithoutProperties(_ref8, ["className"]);
+
+    return _react2.default.createElement(
+        "th",
+        _extends({}, props, { className: "col-method " + className }),
+        "Method"
+    );
+};
+MethodColumn.sortKeyFun = function (flow) {
+    return flow.request.method;
+};
+
+function StatusColumn(_ref9) {
+    var flow = _ref9.flow;
+
+    var status = void 0;
+    if (flow.response) {
+        status = flow.response.status_code;
+    } else {
+        status = null;
     }
-});
+    return _react2.default.createElement(
+        "td",
+        { className: "col-status" },
+        status
+    );
+}
+StatusColumn.Title = function (_ref10) {
+    var _ref10$className = _ref10.className;
+    var className = _ref10$className === undefined ? "" : _ref10$className;
 
-var StatusColumn = _react2.default.createClass({
-    displayName: "StatusColumn",
+    var props = _objectWithoutProperties(_ref10, ["className"]);
 
-    statics: {
-        Title: _react2.default.createClass({
-            displayName: "Title",
+    return _react2.default.createElement(
+        "th",
+        _extends({}, props, { className: "col-status " + className }),
+        "Status"
+    );
+};
+StatusColumn.sortKeyFun = function (flow) {
+    return flow.response ? flow.response.status_code : undefined;
+};
 
-            render: function render() {
-                return _react2.default.createElement(
-                    "th",
-                    _extends({}, this.props, { className: "col-status " + (this.props.className || "") }),
-                    "Status"
-                );
-            }
-        }),
-        sortKeyFun: function sortKeyFun(flow) {
-            return flow.response ? flow.response.status_code : undefined;
-        }
-    },
-    render: function render() {
-        var flow = this.props.flow;
-        var status;
-        if (flow.response) {
-            status = flow.response.status_code;
-        } else {
-            status = null;
-        }
-        return _react2.default.createElement(
-            "td",
-            { className: "col-status" },
-            status
-        );
+function SizeColumn(_ref11) {
+    var flow = _ref11.flow;
+
+    var total = flow.request.contentLength;
+    if (flow.response) {
+        total += flow.response.contentLength || 0;
     }
-});
+    var size = (0, _utils2.formatSize)(total);
+    return _react2.default.createElement(
+        "td",
+        { className: "col-size" },
+        size
+    );
+}
+SizeColumn.Title = function (_ref12) {
+    var _ref12$className = _ref12.className;
+    var className = _ref12$className === undefined ? "" : _ref12$className;
 
-var SizeColumn = _react2.default.createClass({
-    displayName: "SizeColumn",
+    var props = _objectWithoutProperties(_ref12, ["className"]);
 
-    statics: {
-        Title: _react2.default.createClass({
-            displayName: "Title",
-
-            render: function render() {
-                return _react2.default.createElement(
-                    "th",
-                    _extends({}, this.props, { className: "col-size " + (this.props.className || "") }),
-                    "Size"
-                );
-            }
-        }),
-        sortKeyFun: function sortKeyFun(flow) {
-            var total = flow.request.contentLength;
-            if (flow.response) {
-                total += flow.response.contentLength || 0;
-            }
-            return total;
-        }
-    },
-    render: function render() {
-        var flow = this.props.flow;
-
-        var total = flow.request.contentLength;
-        if (flow.response) {
-            total += flow.response.contentLength || 0;
-        }
-        var size = (0, _utils2.formatSize)(total);
-        return _react2.default.createElement(
-            "td",
-            { className: "col-size" },
-            size
-        );
+    return _react2.default.createElement(
+        "th",
+        _extends({}, props, { className: "col-size " + className }),
+        "Size"
+    );
+};
+SizeColumn.sortKeyFun = function (flow) {
+    var total = flow.request.contentLength;
+    if (flow.response) {
+        total += flow.response.contentLength || 0;
     }
-});
+    return total;
+};
 
-var TimeColumn = _react2.default.createClass({
-    displayName: "TimeColumn",
+function TimeColumn(_ref13) {
+    var flow = _ref13.flow;
 
-    statics: {
-        Title: _react2.default.createClass({
-            displayName: "Title",
-
-            render: function render() {
-                return _react2.default.createElement(
-                    "th",
-                    _extends({}, this.props, { className: "col-time " + (this.props.className || "") }),
-                    "Time"
-                );
-            }
-        }),
-        sortKeyFun: function sortKeyFun(flow) {
-            if (flow.response) {
-                return flow.response.timestamp_end - flow.request.timestamp_start;
-            }
-        }
-    },
-    render: function render() {
-        var flow = this.props.flow;
-        var time;
-        if (flow.response) {
-            time = (0, _utils2.formatTimeDelta)(1000 * (flow.response.timestamp_end - flow.request.timestamp_start));
-        } else {
-            time = "...";
-        }
-        return _react2.default.createElement(
-            "td",
-            { className: "col-time" },
-            time
-        );
+    var time = void 0;
+    if (flow.response) {
+        time = (0, _utils2.formatTimeDelta)(1000 * (flow.response.timestamp_end - flow.request.timestamp_start));
+    } else {
+        time = "...";
     }
-});
+    return _react2.default.createElement(
+        "td",
+        { className: "col-time" },
+        time
+    );
+}
+TimeColumn.Title = function (_ref14) {
+    var _ref14$className = _ref14.className;
+    var className = _ref14$className === undefined ? "" : _ref14$className;
+
+    var props = _objectWithoutProperties(_ref14, ["className"]);
+
+    return _react2.default.createElement(
+        "th",
+        _extends({}, props, { className: "col-time " + className }),
+        "Time"
+    );
+};
+TimeColumn.sortKeyFun = function (flow) {
+    return flow.response.timestamp_end - flow.request.timestamp_start;
+};
 
 var all_columns = [TLSColumn, IconColumn, PathColumn, MethodColumn, StatusColumn, SizeColumn, TimeColumn];
 
@@ -1525,12 +1495,17 @@ FlowRow.propTypes = {
     selected: _react2.default.PropTypes.bool
 };
 
-function FlowRow(props) {
-    var flow = props.flow;
+function FlowRow(_ref) {
+    var flow = _ref.flow;
+    var selected = _ref.selected;
+    var highlight = _ref.highlight;
+    var columns = _ref.columns;
+    var selectFlow = _ref.selectFlow;
+
 
     var className = (0, _classnames2.default)({
-        "selected": props.selected,
-        "highlighted": props.highlight && parseFilter(props.highlight)(flow),
+        "selected": selected,
+        "highlighted": highlight && parseFilter(highlight)(flow),
         "intercepted": flow.intercepted,
         "has-request": flow.request,
         "has-response": flow.response
@@ -1539,10 +1514,10 @@ function FlowRow(props) {
     return _react2.default.createElement(
         "tr",
         { className: className, onClick: function onClick() {
-                return props.selectFlow(flow);
+                return selectFlow(flow);
             } },
-        props.columns.map(function (Column) {
-            return _react2.default.createElement(Column, { key: Column.displayName, flow: flow });
+        columns.map(function (Column) {
+            return _react2.default.createElement(Column, { key: Column.name, flow: flow });
         })
     );
 }
@@ -1553,8 +1528,6 @@ var FlowRowContainer = (0, _reactRedux.connect)(function (state, ownProps) {
         highlight: state.flows.highlight,
         selected: state.flows.selected.indexOf(ownProps.flowId) >= 0
     };
-}, function (dispatch, ownProps) {
-    return {};
 })(FlowRow);
 
 var FlowTableHead = function (_React$Component) {
@@ -1608,11 +1581,11 @@ var FlowTableHead = function (_React$Component) {
                 null,
                 this.props.columns.map(function (Column) {
                     return _react2.default.createElement(Column.Title, {
-                        key: Column.displayName,
+                        key: Column.name,
                         onClick: function onClick() {
                             return _this2.onClick(Column);
                         },
-                        className: sortColumn === Column && sortType
+                        className: sortColumn === Column ? sortType : undefined
                     });
                 })
             );
@@ -1769,8 +1742,6 @@ var FlowTableContainer = (0, _reactRedux.connect)(function (state) {
     return {
         flows: state.flows.view
     };
-}, function (dispatch) {
-    return {};
 })(FlowTable);
 
 exports.default = FlowTableContainer;
@@ -4757,22 +4728,23 @@ function makeList(actionType, fetchURL) {
                 list = [].concat(_toConsumableArray(state.list));
                 itemIndex = state.indexOf[action.item.id];
                 list[itemIndex] = action.item;
-                return _extends({}, defaultState, {
-                    list: list
+                return _extends({}, state, {
+                    list: list,
+                    byId: _extends({}, state.byId, _defineProperty({}, action.item.id, action.item))
                 });
 
             case REMOVE:
                 list = [].concat(_toConsumableArray(state.list));
                 itemIndex = state.indexOf[action.item.id];
                 list.splice(itemIndex, 1);
-                return _extends({}, defaultState, {
+                return _extends({}, state, {
                     list: list,
                     byId: _extends({}, state.byId, _defineProperty({}, action.item.id, undefined)),
                     indexOf: _extends({}, state.indexOf, _defineProperty({}, action.item.id, undefined))
                 });
 
             case REQUEST_LIST:
-                return _extends({}, defaultState, {
+                return _extends({}, state, {
                     isFetching: true
                 });
 
@@ -4977,13 +4949,14 @@ function updateViewList(currentView, currentList, nextList, action) {
 
             if (!isInView && shouldBeInView) return sortedInsert(currentView, sortFn, action.item);
             if (isInView && !shouldBeInView) return sortedRemove(currentView, sortFn, action.item);
-            if (isInView && shouldBeInView && sortFn && sortFn(currentItemState) !== sortFn(nextItemState)) {
+            if (isInView && shouldBeInView) {
                 var _ret = function () {
                     var s = [].concat(_toConsumableArray(currentView));
-                    s.sort(makeCompareFn(sortFn));
                     s.indexOf = function (x) {
                         return sortedIndexOf(s, x, sortFn);
                     };
+                    s[s.indexOf(currentItemState)] = nextItemState;
+                    if (sortFn && sortFn(currentItemState) !== sortFn(nextItemState)) s.sort(makeCompareFn(sortFn));
                     return {
                         v: s
                     };

netlib/tcp.py

@@ -580,8 +580,10 @@ class _Connection(object):
 
 @contextlib.contextmanager
 def _closer(client):
-    yield
-    client.close()
+    try:
+        yield
+    finally:
+        client.close()
 
 
 class TCPClient(_Connection):

pathod/language/base.py

@@ -261,7 +261,7 @@ class _Component(Token):
 
     """
         A value component of the primary specification of an message.
-        Components produce byte values desribe the bytes of the message.
+        Components produce byte values describing the bytes of the message.
     """
 
     def values(self, settings):  # pragma: no cover
@@ -272,9 +272,9 @@ class _Component(Token):
 
     def string(self, settings=None):
         """
-            A string representation of the object.
+            A bytestring representation of the object.
         """
-        return "".join(i[:] for i in self.values(settings or {}))
+        return b"".join(i[:] for i in self.values(settings or {}))
 
 
 class KeyValue(_Component):
@@ -391,7 +391,7 @@ class Integer(_Component):
                 "Integer value must be between %s and %s." % self.bounds,
                 0, 0
             )
-        self.value = str(value)
+        self.value = str(value).encode()
 
     @classmethod
     def expr(cls):
@@ -401,10 +401,10 @@ class Integer(_Component):
         return e.setParseAction(lambda x: cls(*x))
 
     def values(self, settings):
-        return self.value
+        return [self.value]
 
     def spec(self):
-        return "%s%s" % (self.preamble, self.value)
+        return "%s%s" % (self.preamble, self.value.decode())
 
     def freeze(self, settings_):
         return self
@@ -555,7 +555,7 @@ class NestedMessage(Token):
         try:
             self.parsed = self.nest_type(
                 self.nest_type.expr().parseString(
-                    value.val,
+                    value.val.decode(),
                     parseAll=True
                 )
             )
@@ -578,4 +578,4 @@ class NestedMessage(Token):
 
     def freeze(self, settings):
         f = self.parsed.freeze(settings).spec()
-        return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f)))
+        return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f.encode())))

pathod/language/generators.py

@@ -3,6 +3,7 @@ import random
 import mmap
 
 import six
+import sys
 
 DATATYPES = dict(
     ascii_letters=string.ascii_letters.encode(),
@@ -68,7 +69,7 @@ class RandomGenerator(object):
     def __getitem__(self, x):
         chars = DATATYPES[self.dtype]
         if isinstance(x, slice):
-            return b"".join(rand_byte(chars) for _ in range(*x.indices(self.length)))
+            return b"".join(rand_byte(chars) for _ in range(*x.indices(min(self.length, sys.maxsize))))
         return rand_byte(chars)
 
     def __repr__(self):

pathod/language/http.py

@@ -57,7 +57,7 @@ class _HeaderMixin(object):
     unique_name = None
 
     def format_header(self, key, value):
-        return [key, ": ", value, "\r\n"]
+        return [key, b": ", value, b"\r\n"]
 
     def values(self, settings):
         return self.format_header(
@@ -88,7 +88,7 @@ class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue):
     def values(self, settings):
         value = self.value.val
         if self.option_used:
-            value = user_agents.get_by_shortcut(value.lower())[2]
+            value = user_agents.get_by_shortcut(value.lower().decode())[2].encode()
 
         return self.format_header(
             self.key.get_generator(settings),
@@ -109,7 +109,7 @@ def get_header(val, headers):
 
 
 class _HTTPMessage(message.Message):
-    version = "HTTP/1.1"
+    version = b"HTTP/1.1"
 
     @property
     def actions(self):
@@ -133,10 +133,10 @@ class _HTTPMessage(message.Message):
 
     def values(self, settings):
         vals = self.preamble(settings)
-        vals.append("\r\n")
+        vals.append(b"\r\n")
         for h in self.headers:
             vals.extend(h.values(settings))
-        vals.append("\r\n")
+        vals.append(b"\r\n")
         if self.body:
             vals.extend(self.body.values(settings))
         return vals
@@ -171,18 +171,18 @@ class Response(_HTTPMessage):
         return self.tok(Reason)
 
     def preamble(self, settings):
-        l = [self.version, " "]
+        l = [self.version, b" "]
         l.extend(self.status_code.values(settings))
         status_code = int(self.status_code.value)
-        l.append(" ")
+        l.append(b" ")
         if self.reason:
             l.extend(self.reason.values(settings))
         else:
             l.append(
                 status_codes.RESPONSES.get(
                     status_code,
-                    "Unknown code"
-                )
+                    b"Unknown code"
+                ).encode()
             )
         return l
 
@@ -205,8 +205,8 @@ class Response(_HTTPMessage):
                 if not get_header(i[0], self.headers):
                     tokens.append(
                         Header(
-                            base.TokValueLiteral(i[0]),
-                            base.TokValueLiteral(i[1]))
+                            base.TokValueLiteral(i[0].decode()),
+                            base.TokValueLiteral(i[1].decode()))
                     )
         if not self.raw:
             if not get_header("Content-Length", self.headers):
@@ -294,11 +294,11 @@ class Request(_HTTPMessage):
 
     def preamble(self, settings):
         v = self.method.values(settings)
-        v.append(" ")
+        v.append(b" ")
         v.extend(self.path.values(settings))
         if self.nested_response:
             v.append(self.nested_response.parsed.spec())
-        v.append(" ")
+        v.append(b" ")
         v.append(self.version)
         return v
 
@@ -314,8 +314,8 @@ class Request(_HTTPMessage):
                 if not get_header(i[0], self.headers):
                     tokens.append(
                         Header(
-                            base.TokValueLiteral(i[0]),
-                            base.TokValueLiteral(i[1])
+                            base.TokValueLiteral(i[0].decode()),
+                            base.TokValueLiteral(i[1].decode())
                         )
                     )
         if not self.raw:

pathod/language/message.py

@@ -1,5 +1,6 @@
 import abc
 from . import actions, exceptions
+from netlib import strutils
 
 LOG_TRUNCATE = 1024
 
@@ -49,7 +50,7 @@ class Message(object):
 
     def preview_safe(self):
         """
-            Return a copy of this message that issafe for previews.
+            Return a copy of this message that is safe for previews.
         """
         tokens = [i for i in self.tokens if not isinstance(i, actions.PauseAt)]
         return self.__class__(tokens)
@@ -80,10 +81,10 @@ class Message(object):
             # We truncate at 1k.
             if hasattr(v, "values"):
                 v = [x[:LOG_TRUNCATE] for x in v.values(settings)]
-                v = "".join(v).encode("string_escape")
+                v = strutils.bytes_to_escaped_str(b"".join(v))
             elif hasattr(v, "__len__"):
                 v = v[:LOG_TRUNCATE]
-                v = v.encode("string_escape")
+                v = strutils.bytes_to_escaped_str(v)
             ret[i] = v
         ret["spec"] = self.spec()
         return ret

pathod/pathoc.py

@@ -247,7 +247,10 @@ class Pathoc(tcp.TCPClient):
 
     def socks_connect(self, connect_to):
         try:
-            client_greet = socks.ClientGreeting(socks.VERSION.SOCKS5, [socks.METHOD.NO_AUTHENTICATION_REQUIRED])
+            client_greet = socks.ClientGreeting(
+                socks.VERSION.SOCKS5,
+                [socks.METHOD.NO_AUTHENTICATION_REQUIRED]
+            )
             client_greet.to_file(self.wfile)
             self.wfile.flush()
 
@@ -286,44 +289,47 @@ class Pathoc(tcp.TCPClient):
         if self.use_http2 and not self.ssl:
             raise NotImplementedError("HTTP2 without SSL is not supported.")
 
-        ret = tcp.TCPClient.connect(self)
+        try:
+            ret = tcp.TCPClient.connect(self)
+            if connect_to:
+                self.http_connect(connect_to)
 
-        if connect_to:
-            self.http_connect(connect_to)
+            self.sslinfo = None
+            if self.ssl:
+                try:
+                    alpn_protos = [b'http/1.1']
+                    if self.use_http2:
+                        alpn_protos.append(b'h2')
 
-        self.sslinfo = None
-        if self.ssl:
-            try:
-                alpn_protos = [b'http/1.1']
-                if self.use_http2:
-                    alpn_protos.append(b'h2')
+                    self.convert_to_ssl(
+                        sni=self.sni,
+                        cert=self.clientcert,
+                        method=self.ssl_version,
+                        options=self.ssl_options,
+                        cipher_list=self.ciphers,
+                        alpn_protos=alpn_protos
+                    )
+                except TlsException as v:
+                    raise PathocError(str(v))
 
-                self.convert_to_ssl(
-                    sni=self.sni,
-                    cert=self.clientcert,
-                    method=self.ssl_version,
-                    options=self.ssl_options,
-                    cipher_list=self.ciphers,
-                    alpn_protos=alpn_protos
+                self.sslinfo = SSLInfo(
+                    self.connection.get_peer_cert_chain(),
+                    self.get_current_cipher(),
+                    self.get_alpn_proto_negotiated()
                 )
-            except TlsException as v:
-                raise PathocError(str(v))
+                if showssl:
+                    print(str(self.sslinfo), file=fp)
 
-            self.sslinfo = SSLInfo(
-                self.connection.get_peer_cert_chain(),
-                self.get_current_cipher(),
-                self.get_alpn_proto_negotiated()
-            )
-            if showssl:
-                print(str(self.sslinfo), file=fp)
+                if self.use_http2:
+                    self.protocol.check_alpn()
+                    if not self.http2_skip_connection_preface:
+                        self.protocol.perform_client_connection_preface()
 
-            if self.use_http2:
-                self.protocol.check_alpn()
-                if not self.http2_skip_connection_preface:
-                    self.protocol.perform_client_connection_preface()
-
-        if self.timeout:
-            self.settimeout(self.timeout)
+            if self.timeout:
+                self.settimeout(self.timeout)
+        except Exception:
+            self.close()
+            raise
         return ret
 
     def stop(self):
@@ -355,6 +361,7 @@ class Pathoc(tcp.TCPClient):
                         return
                 if frm is None:
                     self.ws_framereader.join()
+                    self.ws_framereader = None
                     return
                 yield frm
 

setup.py

@@ -75,7 +75,7 @@ setup(
         "passlib>=1.6.5, <1.7",
         "pyasn1>=0.1.9, <0.2",
         "pyOpenSSL>=16.0, <17.0",
-        "pyparsing>=2.0, <2.1",  # 2.1.1 breaks our binaries, see https://sourceforge.net/p/pyparsing/bugs/93/
+        "pyparsing>=2.1.3, <2.2",
         "pyperclip>=1.5.22, <1.6",
         "requests>=2.9.1, <2.10",
         "six>=1.10, <1.11",

test/pathod/test_language_base.py

@@ -55,8 +55,15 @@ class TestTokValueLiteral:
         v = base.TokValueLiteral("f\x00oo")
         assert v.spec() == repr(v) == r"'f\x00oo'"
 
-        v = base.TokValueLiteral("\"")
-        assert v.spec() == repr(v) == '\'"\''
+        v = base.TokValueLiteral('"')
+        assert v.spec() == repr(v) == """  '"'  """.strip()
+
+        # While pyparsing has a escChar argument for QuotedString,
+        # escChar only performs scapes single-character escapes and does not work for e.g. r"\x02".
+        # Thus, we cannot use that option, which means we cannot have single quotes in strings.
+        # To fix this, we represent single quotes as r"\x07".
+        v = base.TokValueLiteral("'")
+        assert v.spec() == r"'\x27'"
 
     def roundtrip(self, spec):
         e = base.TokValueLiteral.expr()
@@ -311,7 +318,7 @@ def test_options_or_value():
 def test_integer():
     e = base.Integer.expr()
     v = e.parseString("200")[0]
-    assert v.string() == "200"
+    assert v.string() == b"200"
     assert v.spec() == "200"
 
     assert v.freeze({}).value == v.value

test/pathod/test_language_http.py

@@ -1,15 +1,15 @@
-from six.moves import cStringIO as StringIO
+from six import BytesIO
 from pathod import language
 from pathod.language import http, base
 import tutils
 
 
 def parse_request(s):
-    return language.parse_pathoc(s).next()
+    return next(language.parse_pathoc(s))
 
 
 def test_make_error_response():
-    d = StringIO()
+    d = BytesIO()
     s = http.make_error_response("foo")
     language.serve(s, d, {})
 
@@ -24,17 +24,17 @@ class TestRequest:
 
     def test_simple(self):
         r = parse_request('GET:"/foo"')
-        assert r.method.string() == "GET"
-        assert r.path.string() == "/foo"
+        assert r.method.string() == b"GET"
+        assert r.path.string() == b"/foo"
         r = parse_request('GET:/foo')
-        assert r.path.string() == "/foo"
+        assert r.path.string() == b"/foo"
         r = parse_request('GET:@1k')
         assert len(r.path.string()) == 1024
 
     def test_multiple(self):
         r = list(language.parse_pathoc("GET:/ PUT:/"))
-        assert r[0].method.string() == "GET"
-        assert r[1].method.string() == "PUT"
+        assert r[0].method.string() == b"GET"
+        assert r[1].method.string() == b"PUT"
         assert len(r) == 2
 
         l = """
@@ -54,8 +54,8 @@ class TestRequest:
         """
         r = list(language.parse_pathoc(l))
         assert len(r) == 2
-        assert r[0].method.string() == "GET"
-        assert r[1].method.string() == "PUT"
+        assert r[0].method.string() == b"GET"
+        assert r[1].method.string() == b"PUT"
 
         l = """
             get:"http://localhost:9999/p/200":ir,@1
@@ -63,8 +63,8 @@ class TestRequest:
         """
         r = list(language.parse_pathoc(l))
         assert len(r) == 2
-        assert r[0].method.string() == "GET"
-        assert r[1].method.string() == "GET"
+        assert r[0].method.string() == b"GET"
+        assert r[1].method.string() == b"GET"
 
     def test_nested_response(self):
         l = "get:/p:s'200'"
@@ -75,7 +75,7 @@ class TestRequest:
         assert r[0].values({})
 
     def test_render(self):
-        s = StringIO()
+        s = BytesIO()
         r = parse_request("GET:'/foo'")
         assert language.serve(
             r,
@@ -90,8 +90,8 @@ class TestRequest:
             ir,@1
         """
         r = parse_request(l)
-        assert r.method.string() == "GET"
-        assert r.path.string() == "/foo"
+        assert r.method.string() == b"GET"
+        assert r.path.string() == b"/foo"
         assert r.actions
 
         l = """
@@ -106,8 +106,8 @@ class TestRequest:
             ir,@1
         """
         r = parse_request(l)
-        assert r.method.string() == "GET"
-        assert r.path.string().endswith("bar")
+        assert r.method.string() == b"GET"
+        assert r.path.string().endswith(b"bar")
         assert r.actions
 
     def test_spec(self):
@@ -128,66 +128,66 @@ class TestRequest:
     def test_websocket(self):
         r = parse_request('ws:/path/')
         res = r.resolve(language.Settings())
-        assert res.method.string().lower() == "get"
-        assert res.tok(http.Path).value.val == "/path/"
-        assert res.tok(http.Method).value.val.lower() == "get"
-        assert http.get_header("Upgrade", res.headers).value.val == "websocket"
+        assert res.method.string().lower() == b"get"
+        assert res.tok(http.Path).value.val == b"/path/"
+        assert res.tok(http.Method).value.val.lower() == b"get"
+        assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket"
 
         r = parse_request('ws:put:/path/')
         res = r.resolve(language.Settings())
-        assert r.method.string().lower() == "put"
-        assert res.tok(http.Path).value.val == "/path/"
-        assert res.tok(http.Method).value.val.lower() == "put"
-        assert http.get_header("Upgrade", res.headers).value.val == "websocket"
+        assert r.method.string().lower() == b"put"
+        assert res.tok(http.Path).value.val == b"/path/"
+        assert res.tok(http.Method).value.val.lower() == b"put"
+        assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket"
 
 
 class TestResponse:
 
     def dummy_response(self):
-        return language.parse_pathod("400'msg'").next()
+        return next(language.parse_pathod("400'msg'"))
 
     def test_response(self):
-        r = language.parse_pathod("400:m'msg'").next()
-        assert r.status_code.string() == "400"
-        assert r.reason.string() == "msg"
+        r = next(language.parse_pathod("400:m'msg'"))
+        assert r.status_code.string() == b"400"
+        assert r.reason.string() == b"msg"
 
-        r = language.parse_pathod("400:m'msg':b@100b").next()
-        assert r.reason.string() == "msg"
+        r = next(language.parse_pathod("400:m'msg':b@100b"))
+        assert r.reason.string() == b"msg"
         assert r.body.values({})
         assert str(r)
 
-        r = language.parse_pathod("200").next()
-        assert r.status_code.string() == "200"
+        r = next(language.parse_pathod("200"))
+        assert r.status_code.string() == b"200"
         assert not r.reason
-        assert "OK" in [i[:] for i in r.preamble({})]
+        assert b"OK" in [i[:] for i in r.preamble({})]
 
     def test_render(self):
-        s = StringIO()
-        r = language.parse_pathod("400:m'msg'").next()
+        s = BytesIO()
+        r = next(language.parse_pathod("400:m'msg'"))
         assert language.serve(r, s, {})
 
-        r = language.parse_pathod("400:p0,100:dr").next()
+        r = next(language.parse_pathod("400:p0,100:dr"))
         assert "p0" in r.spec()
         s = r.preview_safe()
         assert "p0" not in s.spec()
 
     def test_raw(self):
-        s = StringIO()
-        r = language.parse_pathod("400:b'foo'").next()
+        s = BytesIO()
+        r = next(language.parse_pathod("400:b'foo'"))
         language.serve(r, s, {})
         v = s.getvalue()
-        assert "Content-Length" in v
+        assert b"Content-Length" in v
 
-        s = StringIO()
-        r = language.parse_pathod("400:b'foo':r").next()
+        s = BytesIO()
+        r = next(language.parse_pathod("400:b'foo':r"))
         language.serve(r, s, {})
         v = s.getvalue()
-        assert "Content-Length" not in v
+        assert b"Content-Length" not in v
 
     def test_length(self):
         def testlen(x):
-            s = StringIO()
-            x = x.next()
+            s = BytesIO()
+            x = next(x)
             language.serve(x, s, language.Settings())
             assert x.length(language.Settings()) == len(s.getvalue())
         testlen(language.parse_pathod("400:m'msg':r"))
@@ -196,8 +196,8 @@ class TestResponse:
 
     def test_maximum_length(self):
         def testlen(x):
-            x = x.next()
-            s = StringIO()
+            x = next(x)
+            s = BytesIO()
             m = x.maximum_length({})
             language.serve(x, s, {})
             assert m >= len(s.getvalue())
@@ -225,19 +225,19 @@ class TestResponse:
         tutils.raises("ascii", language.parse_pathod, "foo:b\xf0")
 
     def test_parse_header(self):
-        r = language.parse_pathod('400:h"foo"="bar"').next()
-        assert http.get_header("foo", r.headers)
+        r = next(language.parse_pathod('400:h"foo"="bar"'))
+        assert http.get_header(b"foo", r.headers)
 
     def test_parse_pause_before(self):
-        r = language.parse_pathod("400:p0,10").next()
+        r = next(language.parse_pathod("400:p0,10"))
         assert r.actions[0].spec() == "p0,10"
 
     def test_parse_pause_after(self):
-        r = language.parse_pathod("400:pa,10").next()
+        r = next(language.parse_pathod("400:pa,10"))
         assert r.actions[0].spec() == "pa,10"
 
     def test_parse_pause_random(self):
-        r = language.parse_pathod("400:pr,10").next()
+        r = next(language.parse_pathod("400:pr,10"))
         assert r.actions[0].spec() == "pr,10"
 
     def test_parse_stress(self):
@@ -245,29 +245,29 @@ class TestResponse:
         # returns an int and a python 2.7 int on windows has 32bit precision.
         # Therefore, we should keep the body length < 2147483647 bytes in our
         # tests.
-        r = language.parse_pathod("400:b@1g").next()
+        r = next(language.parse_pathod("400:b@1g"))
         assert r.length({})
 
     def test_spec(self):
         def rt(s):
-            s = language.parse_pathod(s).next().spec()
-            assert language.parse_pathod(s).next().spec() == s
+            s = next(language.parse_pathod(s)).spec()
+            assert next(language.parse_pathod(s)).spec() == s
         rt("400:b@100g")
         rt("400")
         rt("400:da")
 
     def test_websockets(self):
-        r = language.parse_pathod("ws").next()
+        r = next(language.parse_pathod("ws"))
         tutils.raises("no websocket key", r.resolve, language.Settings())
-        res = r.resolve(language.Settings(websocket_key="foo"))
-        assert res.status_code.string() == "101"
+        res = r.resolve(language.Settings(websocket_key=b"foo"))
+        assert res.status_code.string() == b"101"
 
 
 def test_ctype_shortcut():
     e = http.ShortcutContentType.expr()
     v = e.parseString("c'foo'")[0]
-    assert v.key.val == "Content-Type"
-    assert v.value.val == "foo"
+    assert v.key.val == b"Content-Type"
+    assert v.value.val == b"foo"
 
     s = v.spec()
     assert s == e.parseString(s)[0].spec()
@@ -282,8 +282,8 @@ def test_ctype_shortcut():
 def test_location_shortcut():
     e = http.ShortcutLocation.expr()
     v = e.parseString("l'foo'")[0]
-    assert v.key.val == "Location"
-    assert v.value.val == "foo"
+    assert v.key.val == b"Location"
+    assert v.value.val == b"foo"
 
     s = v.spec()
     assert s == e.parseString(s)[0].spec()
@@ -296,23 +296,23 @@ def test_location_shortcut():
 
 
 def test_shortcuts():
-    assert language.parse_pathod(
-        "400:c'foo'").next().headers[0].key.val == "Content-Type"
-    assert language.parse_pathod(
-        "400:l'foo'").next().headers[0].key.val == "Location"
+    assert next(language.parse_pathod(
+        "400:c'foo'")).headers[0].key.val == b"Content-Type"
+    assert next(language.parse_pathod(
+        "400:l'foo'")).headers[0].key.val == b"Location"
 
-    assert "Android" in tutils.render(parse_request("get:/:ua"))
-    assert "User-Agent" in tutils.render(parse_request("get:/:ua"))
+    assert b"Android" in tutils.render(parse_request("get:/:ua"))
+    assert b"User-Agent" in tutils.render(parse_request("get:/:ua"))
 
 
 def test_user_agent():
     e = http.ShortcutUserAgent.expr()
     v = e.parseString("ua")[0]
-    assert "Android" in v.string()
+    assert b"Android" in v.string()
 
     e = http.ShortcutUserAgent.expr()
     v = e.parseString("u'a'")[0]
-    assert "Android" not in v.string()
+    assert b"Android" not in v.string()
 
     v = e.parseString("u@100'")[0]
     assert len(str(v.freeze({}).value)) > 100
@@ -324,7 +324,7 @@ def test_user_agent():
 def test_nested_response():
     e = http.NestedResponse.expr()
     v = e.parseString("s'200'")[0]
-    assert v.value.val == "200"
+    assert v.value.val == b"200"
     tutils.raises(
         language.ParseException,
         e.parseString,
@@ -340,9 +340,7 @@ def test_nested_response():
 def test_nested_response_freeze():
     e = http.NestedResponse(
         base.TokValueLiteral(
-            "200:b'foo':i10,'\\x27'".encode(
-                "string_escape"
-            )
+            r"200:b\'foo\':i10,\'\\x27\'"
         )
     )
     assert e.freeze({})

test/pathod/test_language_websocket.py

@@ -6,7 +6,7 @@ import tutils
 
 
 def parse_request(s):
-    return language.parse_pathoc(s).next()
+    return next(language.parse_pathoc(s))
 
 
 class TestWebsocketFrame:
@@ -93,9 +93,9 @@ class TestWebsocketFrame:
     def test_rawbody(self):
         frm = self.fr("wf:mask:r'foo'")
         assert len(frm.payload) == 3
-        assert frm.payload != "foo"
+        assert frm.payload != b"foo"
 
-        assert self.fr("wf:r'foo'").payload == "foo"
+        assert self.fr("wf:r'foo'").payload == b"foo"
 
     def test_construction_2(self):
         # Simple server frame
@@ -109,7 +109,7 @@ class TestWebsocketFrame:
         assert frm.header.masking_key
         frm = self.fr("wf:b'foo':k'abcd'", is_client=True)
         assert frm.header.mask
-        assert frm.header.masking_key == 'abcd'
+        assert frm.header.masking_key == b'abcd'
 
         # Server frame, mask explicitly set
         frm = self.fr("wf:b'foo':mask")
@@ -117,7 +117,7 @@ class TestWebsocketFrame:
         assert frm.header.masking_key
         frm = self.fr("wf:b'foo':k'abcd'")
         assert frm.header.mask
-        assert frm.header.masking_key == 'abcd'
+        assert frm.header.masking_key == b'abcd'
 
         # Client frame, mask explicitly unset
         frm = self.fr("wf:b'foo':-mask", is_client=True)
@@ -128,7 +128,7 @@ class TestWebsocketFrame:
         assert not frm.header.mask
         # We're reading back a corrupted frame - the first 3 characters of the
         # mask is mis-interpreted as the payload
-        assert frm.payload == "abc"
+        assert frm.payload == b"abc"
 
     def test_knone(self):
         with tutils.raises("expected 4 bytes"):
@@ -138,5 +138,5 @@ class TestWebsocketFrame:
         assert self.fr("wf:l3:b'foo'").header.payload_length == 3
         frm = self.fr("wf:l2:b'foo'")
         assert frm.header.payload_length == 2
-        assert frm.payload == "fo"
+        assert frm.payload == b"fo"
         tutils.raises("expected 1024 bytes", self.fr, "wf:l1024:b'foo'")

test/pathod/test_pathoc.py

@@ -17,43 +17,26 @@ def test_response():
 
 
 class PathocTestDaemon(tutils.DaemonTests):
-    def tval(
-        self,
-        requests,
-        showreq=False,
-        showresp=False,
-        explain=False,
-        showssl=False,
-        hexdump=False,
-        timeout=None,
-        ignorecodes=(),
-        ignoretimeout=None,
-        showsummary=True
-    ):
+    def tval(self, requests, timeout=None, showssl=False, **kwargs):
         s = StringIO()
         c = pathoc.Pathoc(
             ("127.0.0.1", self.d.port),
             ssl=self.ssl,
-            showreq=showreq,
-            showresp=showresp,
-            explain=explain,
-            hexdump=hexdump,
-            ignorecodes=ignorecodes,
-            ignoretimeout=ignoretimeout,
-            showsummary=showsummary,
-            fp=s
+            fp=s,
+            **kwargs
         )
         with c.connect(showssl=showssl, fp=s):
             if timeout:
                 c.settimeout(timeout)
             for i in requests:
                 r = language.parse_pathoc(i).next()
-                if explain:
+                if kwargs.get("explain"):
                     r = r.freeze(language.Settings())
                 try:
                     c.request(r)
                 except NetlibException:
                     pass
+        self.d.wait_for_silence()
         return s.getvalue()
 
 
@@ -66,14 +49,10 @@ class TestDaemonSSL(PathocTestDaemon):
     )
 
     def test_sni(self):
-        c = pathoc.Pathoc(
-            ("127.0.0.1", self.d.port),
-            ssl=True,
-            sni="foobar.com",
-            fp=None
+        self.tval(
+            ["get:/p/200"],
+            sni="foobar.com"
         )
-        with c.connect():
-            c.request("get:/p/200")
         log = self.d.log()
         assert log[0]["request"]["sni"] == "foobar.com"
 
@@ -81,15 +60,10 @@ class TestDaemonSSL(PathocTestDaemon):
         assert "certificate chain" in self.tval(["get:/p/200"], showssl=True)
 
     def test_clientcert(self):
-        c = pathoc.Pathoc(
-            ("127.0.0.1", self.d.port),
-            ssl=True,
+        self.tval(
+            ["get:/p/200"],
             clientcert=tutils.test_data.path("data/clientcert/client.pem"),
-            fp=None
         )
-        with c.connect():
-            c.request("get:/p/200")
-
         log = self.d.log()
         assert log[0]["request"]["clientcert"]["keyinfo"]
 
@@ -170,9 +144,7 @@ class TestDaemon(PathocTestDaemon):
         assert "Invalid server response" in self.tval(["get:'/p/200:d2'"])
 
     def test_websocket_shutdown(self):
-        c = pathoc.Pathoc(("127.0.0.1", self.d.port), fp=None)
-        with c.connect():
-            c.request("ws:/")
+        self.tval(["ws:/"])
 
     def test_wait_finish(self):
         c = pathoc.Pathoc(
@@ -182,9 +154,10 @@ class TestDaemon(PathocTestDaemon):
         )
         with c.connect():
             c.request("ws:/")
-            c.request("wf:f'wf:x100'")
-            [i for i in c.wait(timeout=0, finish=False)]
-            [i for i in c.wait(timeout=0)]
+            c.request("wf:f'wf'")
+            # This should read a frame and close the websocket reader
+            assert len([i for i in c.wait(timeout=5, finish=False)]) == 1
+            assert not [i for i in c.wait(timeout=0)]
 
     def test_connect_fail(self):
         to = ("foobar", 80)

test/pathod/test_pathod.py

@@ -145,14 +145,14 @@ class CommonTests(tutils.DaemonTests):
 
     def test_invalid_first_line(self):
         c = tcp.TCPClient(("localhost", self.d.port))
-        c.connect()
-        if self.ssl:
-            c.convert_to_ssl()
-        c.wfile.write("foo\n\n\n")
-        c.wfile.flush()
-        l = self.d.last_log()
-        assert l["type"] == "error"
-        assert "foo" in l["msg"]
+        with c.connect():
+            if self.ssl:
+                c.convert_to_ssl()
+            c.wfile.write("foo\n\n\n")
+            c.wfile.flush()
+            l = self.d.last_log()
+            assert l["type"] == "error"
+            assert "foo" in l["msg"]
 
     def test_invalid_content_length(self):
         tutils.raises(
@@ -238,12 +238,12 @@ class TestDaemonSSL(CommonTests):
         c = tcp.TCPClient(("localhost", self.d.port))
         c.rbufsize = 0
         c.wbufsize = 0
-        c.connect()
-        c.wfile.write("\0\0\0\0")
-        tutils.raises(TlsException, c.convert_to_ssl)
-        l = self.d.last_log()
-        assert l["type"] == "error"
-        assert "SSL" in l["msg"]
+        with c.connect():
+            c.wfile.write("\0\0\0\0")
+            tutils.raises(TlsException, c.convert_to_ssl)
+            l = self.d.last_log()
+            assert l["type"] == "error"
+            assert "SSL" in l["msg"]
 
     def test_ssl_cipher(self):
         r, _ = self.pathoc([r"get:/p/202"])

test/pathod/tutils.py

@@ -3,6 +3,7 @@ import re
 import shutil
 import requests
 from six.moves import cStringIO as StringIO
+from six import BytesIO
 import urllib
 
 from netlib import tcp
@@ -147,6 +148,6 @@ test_data = utils.Data(__name__)
 
 def render(r, settings=language.Settings()):
     r = r.resolve(settings)
-    s = StringIO()
+    s = BytesIO()
     assert language.serve(r, s, settings)
     return s.getvalue()

tox.ini

@@ -8,7 +8,7 @@ deps = -rrequirements.txt
 commands = py.test -n 8 --timeout 60 ./test
 
 [testenv:py35]
-commands = py.test -n 8 --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py
+commands = py.test -n 8 --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py
 
 [testenv:lint]
 deps = flake8

web/package.json

@@ -21,8 +21,8 @@
     "flux": "^2.1.1",
     "jquery": "^2.2.3",
     "lodash": "^4.11.2",
-    "react": "^15.0.2",
-    "react-dom": "^15.0.2",
+    "react": "^15.1.0",
+    "react-dom": "^15.1.0",
     "react-redux": "^4.4.5",
     "react-router": "^2.4.0",
     "redux": "^3.5.2",
@@ -53,7 +53,6 @@
     "gulp-sourcemaps": "^1.6.0",
     "gulp-util": "^3.0.7",
     "jest": "^0.1.40",
-    "redux-thunk": "^2.1.0",
     "uglifyify": "^3.0.1",
     "vinyl-buffer": "^1.0.0",
     "vinyl-source-stream": "^1.1.0",

web/src/js/components/flowtable-columns.js

@@ -1,190 +1,121 @@
-import React from "react";
-import {RequestUtils, ResponseUtils} from "../flow/utils.js";
-import {formatSize, formatTimeDelta} from "../utils.js";
+import React from "react"
+import {RequestUtils, ResponseUtils} from "../flow/utils.js"
+import {formatSize, formatTimeDelta} from "../utils.js"
 
-var TLSColumn = React.createClass({
-    statics: {
-        Title: React.createClass({
-            render: function(){
-                return <th {...this.props} className={"col-tls " + (this.props.className || "") }></th>;
-            }
-        }),
-        sortKeyFun: function(flow){
-            return flow.request.scheme;
-        }
-    },
-    render: function () {
-        var flow = this.props.flow;
-        var ssl = (flow.request.scheme === "https");
-        var classes;
-        if (ssl) {
-            classes = "col-tls col-tls-https";
-        } else {
-            classes = "col-tls col-tls-http";
-        }
-        return <td className={classes}></td>;
+
+function TLSColumn({flow}) {
+    let ssl = (flow.request.scheme === "https")
+    let classes
+    if (ssl) {
+        classes = "col-tls col-tls-https"
+    } else {
+        classes = "col-tls col-tls-http"
     }
-});
+    return <td className={classes}></td>
+}
+TLSColumn.Title = ({className = "", ...props}) => <th {...props} className={"col-tls " + className }></th>
+TLSColumn.sortKeyFun = flow => flow.request.scheme
 
 
-var IconColumn = React.createClass({
-    statics: {
-        Title: React.createClass({
-            render: function(){
-                return <th {...this.props} className={"col-icon " + (this.props.className || "") }></th>;
-            }
-        })
-    },
-    render: function () {
-        var flow = this.props.flow;
+function IconColumn({flow}) {
+    let icon
+    if (flow.response) {
+        var contentType = ResponseUtils.getContentType(flow.response)
 
-        var icon;
-        if (flow.response) {
-            var contentType = ResponseUtils.getContentType(flow.response);
-
-            //TODO: We should assign a type to the flow somewhere else.
-            if (flow.response.status_code === 304) {
-                icon = "resource-icon-not-modified";
-            } else if (300 <= flow.response.status_code && flow.response.status_code < 400) {
-                icon = "resource-icon-redirect";
-            } else if (contentType && contentType.indexOf("image") >= 0) {
-                icon = "resource-icon-image";
-            } else if (contentType && contentType.indexOf("javascript") >= 0) {
-                icon = "resource-icon-js";
-            } else if (contentType && contentType.indexOf("css") >= 0) {
-                icon = "resource-icon-css";
-            } else if (contentType && contentType.indexOf("html") >= 0) {
-                icon = "resource-icon-document";
-            }
+        //TODO: We should assign a type to the flow somewhere else.
+        if (flow.response.status_code === 304) {
+            icon = "resource-icon-not-modified"
+        } else if (300 <= flow.response.status_code && flow.response.status_code < 400) {
+            icon = "resource-icon-redirect"
+        } else if (contentType && contentType.indexOf("image") >= 0) {
+            icon = "resource-icon-image"
+        } else if (contentType && contentType.indexOf("javascript") >= 0) {
+            icon = "resource-icon-js"
+        } else if (contentType && contentType.indexOf("css") >= 0) {
+            icon = "resource-icon-css"
+        } else if (contentType && contentType.indexOf("html") >= 0) {
+            icon = "resource-icon-document"
         }
-        if (!icon) {
-            icon = "resource-icon-plain";
-        }
-
-
-        icon += " resource-icon";
-        return <td className="col-icon">
-            <div className={icon}></div>
-        </td>;
     }
-});
-
-var PathColumn = React.createClass({
-    statics: {
-        Title: React.createClass({
-            render: function(){
-                return <th {...this.props} className={"col-path " + (this.props.className || "") }>Path</th>;
-            }
-        }),
-        sortKeyFun: function(flow){
-            return RequestUtils.pretty_url(flow.request);
-        }
-    },
-    render: function () {
-        var flow = this.props.flow;
-        return <td className="col-path">
-            {flow.request.is_replay ? <i className="fa fa-fw fa-repeat pull-right"></i> : null}
-            {flow.intercepted ? <i className="fa fa-fw fa-pause pull-right"></i> : null}
-            { RequestUtils.pretty_url(flow.request) }
-        </td>;
+    if (!icon) {
+        icon = "resource-icon-plain"
     }
-});
+
+    icon += " resource-icon"
+    return <td className="col-icon">
+        <div className={icon}></div>
+    </td>
+}
+IconColumn.Title = ({className = "", ...props}) => <th {...props} className={"col-icon " + className }></th>
 
 
-var MethodColumn = React.createClass({
-    statics: {
-        Title: React.createClass({
-            render: function(){
-                return <th {...this.props} className={"col-method " + (this.props.className || "") }>Method</th>;
-            }
-        }),
-        sortKeyFun: function(flow){
-            return flow.request.method;
-        }
-    },
-    render: function () {
-        var flow = this.props.flow;
-        return <td className="col-method">{flow.request.method}</td>;
+function PathColumn({flow}) {
+    return <td className="col-path">
+        {flow.request.is_replay ? <i className="fa fa-fw fa-repeat pull-right"></i> : null}
+        {flow.intercepted ? <i className="fa fa-fw fa-pause pull-right"></i> : null}
+        { RequestUtils.pretty_url(flow.request) }
+    </td>
+}
+PathColumn.Title = ({className = "", ...props}) =>
+    <th {...props} className={"col-path " + className }>Path</th>
+PathColumn.sortKeyFun = flow => RequestUtils.pretty_url(flow.request)
+
+
+function MethodColumn({flow}) {
+    return <td className="col-method">{flow.request.method}</td>
+}
+MethodColumn.Title = ({className = "", ...props}) =>
+    <th {...props} className={"col-method " + className }>Method</th>
+MethodColumn.sortKeyFun = flow => flow.request.method
+
+
+function StatusColumn({flow}) {
+    let status
+    if (flow.response) {
+        status = flow.response.status_code
+    } else {
+        status = null
     }
-});
+    return <td className="col-status">{status}</td>
+
+}
+StatusColumn.Title = ({className = "", ...props}) =>
+    <th {...props} className={"col-status " + className }>Status</th>
+StatusColumn.sortKeyFun = flow => flow.response ? flow.response.status_code : undefined
 
 
-var StatusColumn = React.createClass({
-    statics: {
-        Title: React.createClass({
-            render: function(){
-                return <th {...this.props} className={"col-status " + (this.props.className || "") }>Status</th>;
-            }
-        }),
-        sortKeyFun: function(flow){
-            return flow.response ? flow.response.status_code : undefined;
-        }
-    },
-    render: function () {
-        var flow = this.props.flow;
-        var status;
-        if (flow.response) {
-            status = flow.response.status_code;
-        } else {
-            status = null;
-        }
-        return <td className="col-status">{status}</td>;
+function SizeColumn({flow}) {
+    let total = flow.request.contentLength
+    if (flow.response) {
+        total += flow.response.contentLength || 0
     }
-});
+    let size = formatSize(total)
+    return <td className="col-size">{size}</td>
 
-
-var SizeColumn = React.createClass({
-    statics: {
-        Title: React.createClass({
-            render: function(){
-                return <th {...this.props} className={"col-size " + (this.props.className || "") }>Size</th>;
-            }
-        }),
-        sortKeyFun: function(flow){
-            var total = flow.request.contentLength;
-            if (flow.response) {
-                total += flow.response.contentLength || 0;
-            }
-            return total;
-        }
-    },
-    render: function () {
-        var flow = this.props.flow;
-
-        var total = flow.request.contentLength;
-        if (flow.response) {
-            total += flow.response.contentLength || 0;
-        }
-        var size = formatSize(total);
-        return <td className="col-size">{size}</td>;
+}
+SizeColumn.Title = ({className = "", ...props}) =>
+    <th {...props} className={"col-size " + className }>Size</th>
+SizeColumn.sortKeyFun = flow => {
+    let total = flow.request.contentLength
+    if (flow.response) {
+        total += flow.response.contentLength || 0
     }
-});
+    return total
+}
 
 
-var TimeColumn = React.createClass({
-    statics: {
-        Title: React.createClass({
-            render: function(){
-                return <th {...this.props} className={"col-time " + (this.props.className || "") }>Time</th>;
-            }
-        }),
-        sortKeyFun: function(flow){
-            if(flow.response) {
-                return flow.response.timestamp_end - flow.request.timestamp_start;
-            }
-        }
-    },
-    render: function () {
-        var flow = this.props.flow;
-        var time;
-        if (flow.response) {
-            time = formatTimeDelta(1000 * (flow.response.timestamp_end - flow.request.timestamp_start));
-        } else {
-            time = "...";
-        }
-        return <td className="col-time">{time}</td>;
+function TimeColumn({flow}) {
+    let time
+    if (flow.response) {
+        time = formatTimeDelta(1000 * (flow.response.timestamp_end - flow.request.timestamp_start))
+    } else {
+        time = "..."
     }
-});
+    return <td className="col-time">{time}</td>
+}
+TimeColumn.Title = ({className = "", ...props}) =>
+    <th {...props} className={"col-time " + className }>Time</th>
+TimeColumn.sortKeyFun = flow => flow.response.timestamp_end - flow.request.timestamp_start
 
 
 var all_columns = [
@@ -195,6 +126,6 @@ var all_columns = [
     StatusColumn,
     SizeColumn,
     TimeColumn
-];
+]
 
-export default all_columns;
+export default all_columns

web/src/js/components/flowtable.js

@@ -19,21 +19,20 @@ FlowRow.propTypes = {
     selected: React.PropTypes.bool,
 };
 
-function FlowRow(props) {
-    const flow = props.flow;
+function FlowRow({flow, selected, highlight, columns, selectFlow}) {
 
     const className = classNames({
-        "selected": props.selected,
-        "highlighted": props.highlight && parseFilter(props.highlight)(flow),
+        "selected": selected,
+        "highlighted": highlight && parseFilter(highlight)(flow),
         "intercepted": flow.intercepted,
         "has-request": flow.request,
         "has-response": flow.response,
     });
 
     return (
-        <tr className={className} onClick={() => props.selectFlow(flow)}>
-            {props.columns.map(Column => (
-                <Column key={Column.displayName} flow={flow}/>
+        <tr className={className} onClick={() => selectFlow(flow)}>
+            {columns.map(Column => (
+                <Column key={Column.name} flow={flow}/>
             ))}
         </tr>
     );
@@ -44,11 +43,8 @@ const FlowRowContainer = connect(
         flow: state.flows.all.byId[ownProps.flowId],
         highlight: state.flows.highlight,
         selected: state.flows.selected.indexOf(ownProps.flowId) >= 0
-    }),
-    (dispatch, ownProps) => ({
-
     })
-)(FlowRow);
+)(FlowRow)
 
 class FlowTableHead extends React.Component {
 
@@ -59,7 +55,7 @@ class FlowTableHead extends React.Component {
 
     constructor(props, context) {
         super(props, context);
-        this.state = { sortColumn: undefined, sortDesc: false };
+        this.state = {sortColumn: undefined, sortDesc: false};
     }
 
     onClick(Column) {
@@ -69,20 +65,20 @@ class FlowTableHead extends React.Component {
 
         if (Column === this.state.sortColumn) {
             sortDesc = !sortDesc;
-            this.setState({ sortDesc });
+            this.setState({sortDesc});
         } else {
-            this.setState({ sortColumn: hasSort && Column, sortDesc: false });
+            this.setState({sortColumn: hasSort && Column, sortDesc: false});
         }
 
         let sortKeyFun = Column.sortKeyFun;
         if (sortDesc) {
-            sortKeyFun = hasSort && function() {
-                const k = Column.sortKeyFun.apply(this, arguments);
-                if (_.isString(k)) {
-                    return reverseString("" + k);
-                }
-                return -k;
-            };
+            sortKeyFun = hasSort && function () {
+                    const k = Column.sortKeyFun.apply(this, arguments);
+                    if (_.isString(k)) {
+                        return reverseString("" + k);
+                    }
+                    return -k;
+                };
         }
 
         this.props.setSortKeyFun(sortKeyFun);
@@ -95,9 +91,9 @@ class FlowTableHead extends React.Component {
             <tr>
                 {this.props.columns.map(Column => (
                     <Column.Title
-                        key={Column.displayName}
+                        key={Column.name}
                         onClick={() => this.onClick(Column)}
-                        className={sortColumn === Column && sortType}
+                        className={sortColumn === Column ? sortType : undefined}
                     />
                 ))}
             </tr>
@@ -118,7 +114,7 @@ class FlowTable extends React.Component {
     constructor(props, context) {
         super(props, context);
 
-        this.state = { vScroll: calcVScroll() };
+        this.state = {vScroll: calcVScroll()};
 
         this.onViewportUpdate = this.onViewportUpdate.bind(this);
     }
@@ -132,7 +128,7 @@ class FlowTable extends React.Component {
     }
 
     componentWillReceiveProps(nextProps) {
-        if(nextProps.selected && nextProps.selected !== this.props.selected){
+        if (nextProps.selected && nextProps.selected !== this.props.selected) {
             window.setTimeout(() => this.scrollIntoView(nextProps.selected), 1)
         }
     }
@@ -154,7 +150,7 @@ class FlowTable extends React.Component {
 
         if (!shallowEqual(this.state.vScroll, vScroll) ||
             this.state.viewportTop !== viewportTop) {
-            this.setState({ vScroll, viewportTop });
+            this.setState({vScroll, viewportTop});
         }
     }
 
@@ -190,22 +186,22 @@ class FlowTable extends React.Component {
             <div className="flow-table" onScroll={this.onViewportUpdate}>
                 <table>
                     <thead ref="head" style={{ transform }}>
-                        <FlowTableHead
-                            columns={flowtable_columns}
-                            setSortKeyFun={this.props.setSortKeyFun}
-                        />
+                    <FlowTableHead
+                        columns={flowtable_columns}
+                        setSortKeyFun={this.props.setSortKeyFun}
+                    />
                     </thead>
                     <tbody>
-                        <tr style={{ height: vScroll.paddingTop }}></tr>
-                        {flows.map(flow => (
-                            <FlowRowContainer
-                                key={flow.id}
-                                flowId={flow.id}
-                                columns={flowtable_columns}
-                                selectFlow={this.props.selectFlow}
-                            />
-                        ))}
-                        <tr style={{ height: vScroll.paddingBottom }}></tr>
+                    <tr style={{ height: vScroll.paddingTop }}></tr>
+                    {flows.map(flow => (
+                        <FlowRowContainer
+                            key={flow.id}
+                            flowId={flow.id}
+                            columns={flowtable_columns}
+                            selectFlow={this.props.selectFlow}
+                        />
+                    ))}
+                    <tr style={{ height: vScroll.paddingBottom }}></tr>
                     </tbody>
                 </table>
             </div>
@@ -221,8 +217,6 @@ const parseFilter = _.memoize(Filt.parse)
 const FlowTableContainer = connect(
     state => ({
         flows: state.flows.view,
-    }),
-    dispatch => ({
     })
 )(FlowTable)
 

web/src/js/ducks/utils/list.js

@@ -62,8 +62,9 @@ export default function makeList(actionType, fetchURL) {
                 itemIndex = state.indexOf[action.item.id]
                 list[itemIndex] = action.item
                 return {
-                    ...defaultState,
-                    list
+                    ...state,
+                    list,
+                    byId: {...state.byId, [action.item.id]: action.item},
                 }
 
             case REMOVE:
@@ -71,7 +72,7 @@ export default function makeList(actionType, fetchURL) {
                 itemIndex = state.indexOf[action.item.id]
                 list.splice(itemIndex, 1)
                 return {
-                    ...defaultState,
+                    ...state,
                     list,
                     byId: {...state.byId, [action.item.id]: undefined},
                     indexOf: {...state.indexOf, [action.item.id]: undefined},
@@ -79,7 +80,7 @@ export default function makeList(actionType, fetchURL) {
 
             case REQUEST_LIST:
                 return {
-                    ...defaultState,
+                    ...state,
                     isFetching: true
                 }
 

web/src/js/ducks/utils/view.js

@@ -43,12 +43,8 @@ const sortedRemove = (list, sortFn, item) => {
 }
 
 export function sortedIndexOf(list, value, sortFn) {
-    if (sortFn === false){
-        let i = 0
-        while (i < list.length && list[i].id !== value.id){
-            i++
-        }
-        return i
+    if (!sortFn) {
+        sortFn = x => 0 // This triggers the linear search for flows that have the same sort value.
     }
 
     let low = 0,
@@ -57,7 +53,7 @@ export function sortedIndexOf(list, value, sortFn) {
         mid;
     while (low < high) {
         mid = (low + high) >>> 1;
-        if ((sortFn(list[mid]) < val) ) {
+        if (sortFn(list[mid]) < val) {
             low = mid + 1
         } else {
             high = mid
@@ -96,10 +92,12 @@ export function updateViewList(currentView, currentList, nextList, action, filte
                 return sortedInsert(currentView, sortFn, action.item)
             if (isInView && !shouldBeInView)
                 return sortedRemove(currentView, sortFn, action.item)
-            if (isInView && shouldBeInView && sortFn && sortFn(currentItemState) !== sortFn(nextItemState)) {
+            if (isInView && shouldBeInView) {
                 let s = [...currentView]
-                s.sort(makeCompareFn(sortFn))
                 s.indexOf = x => sortedIndexOf(s, x, sortFn)
+                s[s.indexOf(currentItemState)] = nextItemState
+                if (sortFn && sortFn(currentItemState) !== sortFn(nextItemState))
+                    s.sort(makeCompareFn(sortFn))
                 return s
             }
             return currentView
@@ -123,4 +121,4 @@ export function updateViewFilter(list, filterFn = defaultFilterFn, sortFn = defa
     filtered.indexOf = x => sortedIndexOf(filtered, x, sortFn)
 
     return filtered
-}
+}