Merge branch 'master' into 0.10

.travis.yml

@@ -0,0 +1,15 @@
+language: python
+python:
+  - "2.7"
+# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
+install: 
+  - "pip install coveralls --use-mirrors"
+  - "pip install nose-cov --use-mirrors"
+  - "pip install -r requirements.txt --use-mirrors"
+  - "pip install --upgrade git+https://github.com/mitmproxy/netlib.git"
+  - "pip install --upgrade git+https://github.com/mitmproxy/pathod.git"
+# command to run tests, e.g. python setup.py test
+script: 
+  - "nosetests --with-cov --cov-report term-missing"
+after_success: 
+  - coveralls

CHANGELOG

@@ -1,3 +1,25 @@
+25 August 2013: mitmproxy 0.9.2:
+
+    * Improvements to the mitmproxywrapper.py helper script for OSX.
+    
+    * Don't take minor version into account when checking for serialized file
+      compatibility.
+
+    * Fix a bug causing resource exhaustion under some circumstances for SSL
+      connections.
+
+    * Revamp the way we store interception certificates. We used to store these
+      on disk, they're now in-memory. This fixes a race condition related to
+      cert handling, and improves compatibility with Windows, where the rules
+      governing permitted file names are weird, resulting in errors for some
+      valid IDNA-encoded names.
+
+    * Display transfer rates for responses in the flow list.
+    
+    * Many other small bugfixes and improvements.
+    
+
+
 
 16 June 2013: mitmproxy 0.9.1:
 

CONTRIBUTORS

@@ -1,39 +1,42 @@
-   777	Aldo Cortesi
+   801	Aldo Cortesi
     18	Henrik Nordstrom
     13	Thomas Roth
+    13	Maximilian Hils
     11	Stephen Altamirano
     10	András Veres-Szentkirályi
-     8	Rouli
      8	Jason A. Novak
+     8	Rouli
      7	Alexis Hildebrandt
-     6	Maximilian Hils
+     4	Marc Liyanage
      4	Valtteri Virtanen
      4	Bryan Bishop
      3	Chris Neasbitt
-     2	Heikki Hannikainen
+     3	Kyle Manna
      2	Jim Lloyd
-     2	Mark E. Haase
+     2	Matthias Urlichs
      2	Michael Frister
-     2	Rob Wills
      2	alts
+     2	Rob Wills
      2	israel
-     1	Mathieu Mitchell
-     1	Michael Bisbjerg
-     1	capt8bit
-     1	Nicolas Esteves
+     2	Mark E. Haase
+     2	Heikki Hannikainen
+     1	Oleksandr Sheremet
      1	Paul
-     1	phil plante
      1	Rory McCann
-     1	Jakub Nawalaniec
      1	Rune Halvorsen
      1	Sahn Lam
-     1	Ivaylo Popov
-     1	Andy Smith
-     1	Ulrich Petri
-     1	Henrik Nordström
      1	Felix Wolfsteller
-     1	Yuangxuan Wang
-     1	Kit Randel
-     1	Marc Liyanage
-     1	meeee
      1	Eric Entzel
+     1	Ulrich Petri
+     1	Andy Smith
+     1	Yuangxuan Wang
+     1	capt8bit
+     1	meeee
+     1	Jakub Nawalaniec
+     1	Kit Randel
+     1	phil plante
+     1	Ivaylo Popov
+     1	Mathieu Mitchell
+     1	Henrik Nordström
+     1	Michael Bisbjerg
+     1	Nicolas Esteves

README.mkd

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/mitmproxy/mitmproxy.png)](https://travis-ci.org/mitmproxy/mitmproxy) [![Coverage Status](https://coveralls.io/repos/mitmproxy/mitmproxy/badge.png)](https://coveralls.io/r/mitmproxy/mitmproxy)
+
 __mitmproxy__ is an interactive, SSL-capable man-in-the-middle proxy for HTTP
 with a console interface.
 

libmproxy/console/common.py

@@ -144,6 +144,8 @@ def raw_format_flow(f, focus, extended, padding):
         if f["resp_ctype"]:
             resp.append(fcol(f["resp_ctype"], rc))
         resp.append(fcol(f["resp_clen"], rc))
+        resp.append(fcol(f["resp_rate"], rc))
+
     elif f["err_msg"]:
         resp.append(fcol(SYMBOL_RETURN, "error"))
         resp.append(
@@ -185,11 +187,17 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
             contentdesc = "[content missing]"
         else:
             contentdesc = "[no content]"
+
+        delta = f.response.timestamp_end - f.response.timestamp_start
+        size = len(f.response.content) + f.response.get_header_size()
+        rate = utils.pretty_size(size / delta)
+
         d.update(dict(
             resp_code = f.response.code,
             resp_is_replay = f.response.is_replay(),
             resp_acked = f.response.reply.acked,
-            resp_clen = contentdesc
+            resp_clen = contentdesc,
+            resp_rate = "{0}/s".format(rate),
         ))
         t = f.response.headers["content-type"]
         if t:

libmproxy/flow.py

@@ -618,7 +618,7 @@ class Response(HTTPMsg):
         self.headers, self.content = headers, content
         self.cert = cert
         self.timestamp_start = timestamp_start or utils.timestamp()
-        self.timestamp_end = max(timestamp_end or utils.timestamp(), timestamp_start)
+        self.timestamp_end = timestamp_end or utils.timestamp()
         self.replay = False
 
     def _refresh_cookie(self, c, delta):

libmproxy/proxy.py

@@ -88,11 +88,13 @@ class RequestReplayThread(threading.Thread):
             server = ServerConnection(self.config, r.scheme, r.host, r.port, r.host)
             server.connect()
             server.send(r)
+            tsstart = utils.timestamp()
             httpversion, code, msg, headers, content = http.read_response(
                 server.rfile, r.method, self.config.body_size_limit
             )
             response = flow.Response(
-                self.flow.request, httpversion, code, msg, headers, content, server.cert
+                self.flow.request, httpversion, code, msg, headers, content, server.cert, 
+                server.rfile.first_byte_timestamp
             )
             self.channel.ask(response)
         except (ProxyError, http.HttpError, tcp.NetLibError), v:
@@ -224,6 +226,7 @@ class ProxyHandler(tcp.BaseHandler):
                             request.ssl_setup_timestamp = sc.ssl_setup_timestamp
                         sc.rfile.reset_timestamps()
                         try:
+                            tsstart = utils.timestamp()
                             httpversion, code, msg, headers, content = http.read_response(
                                 sc.rfile,
                                 request.method,
@@ -242,7 +245,7 @@ class ProxyHandler(tcp.BaseHandler):
 
                     response = flow.Response(
                         request, httpversion, code, msg, headers, content, sc.cert,
-                        sc.rfile.first_byte_timestamp, utils.timestamp()
+                        sc.rfile.first_byte_timestamp
                     )
                     response_reply = self.channel.ask(response)
                     # Not replying to the server invalidates the server

libmproxy/utils.py

@@ -74,7 +74,7 @@ def pretty_size(size):
     suffixes = [
         ("B",   2**10),
         ("kB",   2**20),
-        ("M",   2**30),
+        ("MB",   2**30),
     ]
     for suf, lim in suffixes:
         if size >= lim:

libmproxy/version.py

@@ -1,4 +1,4 @@
-IVERSION = (0, 9, 1)
+IVERSION = (0, 9, 2)
 VERSION = ".".join(str(i) for i in IVERSION)
 NAME = "mitmproxy"
 NAMEVERSION = NAME + " " + VERSION

release/osx-binaries

@@ -2,8 +2,15 @@
 
 # Quick and dangerous script for building OSX binaries.
 
-# First, have a recent checkout of the dev version of pyinstaller. Change into
-# the pyinstaller directory, and then run this script.
+# A few quirks to note, which should be re-checked every release:
+# - We require the latest development version of PyInstaller.
+ 
+# - PyInstaller has trouble detecting the zope.interfaces package. This is
+# required by Twisted, which for mysterious reasons is required by Urwid. The
+# answer is to touch the __init__.py file in the zope directory. On my system:
+#   touch /Library/Python/2.7/site-packages/zope/__init__.py
+
+# To run, change into the pyinstaller directory, and then run this script.
 
 DST=/tmp/osx-mitmproxy
 MITMPROXY=~/mitmproxy/mitmproxy

requirements.txt

@@ -1,14 +1,15 @@
-Flask==0.9
-Jinja2==2.7
-MarkupSafe==0.18
-PIL==1.1.7
-Werkzeug==0.8.3
-lxml==3.2.1
-netlib==0.9
-nose==1.3.0
-pathod==0.9
-pyOpenSSL==0.13
-pyasn1==0.1.7
-requests==1.2.2
-urwid==1.1.1
-wsgiref==0.1.2
+Flask>=0.9
+Jinja2>=2.7
+MarkupSafe>=0.18
+PIL>=1.1.7
+Werkzeug>=0.8.3
+lxml>=3.2.1
+netlib>=0.9.2
+nose>=1.3.0
+pathod>=0.9.2
+pyOpenSSL>=0.13
+pyasn1>=0.1.7
+requests>=1.2.2
+urwid>=1.1.1
+wsgiref>=0.1.2
+jsbeautifier>=1.4.0

test/test_utils.py

@@ -38,7 +38,7 @@ def test_pretty_size():
     assert utils.pretty_size(100) == "100B"
     assert utils.pretty_size(1024) == "1kB"
     assert utils.pretty_size(1024 + (1024/2.0)) == "1.5kB"
-    assert utils.pretty_size(1024*1024) == "1M"
+    assert utils.pretty_size(1024*1024) == "1MB"
 
 
 def test_pkg_data():