Merge remote-tracking branch 'upstream/master' into print-bracket-fix

Conflicts:
	examples/har_extractor.py
	examples/nonblocking.py
	examples/read_dumpfile
	libmproxy/web/app.py

.env

@@ -0,0 +1,5 @@
+DIR="${0%/*}"
+if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then
+    echo "Activating mitmproxy virtualenv..."
+    source "$DIR/../venv.mitmproxy/bin/activate"
+fi

CHANGELOG

@@ -1,3 +1,43 @@
+18 May 2015: mitmproxy 0.12
+
+    * mitmproxy console: Significant revamp of the UI. The major changes are
+      listed below, and in addition almost every aspect of the UI has
+      been tweaked, and performance has improved significantly.
+
+    * mitmproxy console: A new options screen has been created ("o" shortcut),
+      and many options that were previously manipulated directly via a
+      keybinding have been moved there.
+
+    * mitmproxy console: Big improvement in palettes. This includes improvements
+      to all colour schemes. Palettes now set the terminal background colour by
+      default, and a new --palette-transparent option has been added to disable
+      this.
+
+    * mitmproxy console: g/G shortcuts throughout mitmproxy console to jump
+      to the beginning/end of the current view.
+
+    * mitmproxy console: switch  palettes on the fly from the options screen.
+
+    * mitmproxy console: A cookie editor has been added for mitmproxy console
+      at long last.
+
+    * mitmproxy console: Various components of requests and responses can be
+      copied to the clipboard from mitmproxy - thanks to @marceloglezer.
+
+    * Support for creating new requests from scratch in mitmproxy console (@marceloglezer).
+
+    * SSLKEYLOGFILE environment variable to specify a logging location for TLS
+      master keys. This can be used with tools like Wireshark to allow TLS
+      decoding.
+
+    * Server facing SSL cipher suite specification (thanks to Jim Shaver).
+
+    * Official support for transparent proxying on FreeBSD - thanks to Mike C
+      (http://github.com/mike-pt).
+
+    * Many other small bugfixes and improvemenets throughout the project.
+
+
 29 Dec 2014: mitmproxy 0.11.2:
 
     * Configuration files - mitmproxy.conf, mitmdump.conf, common.conf in the

CONTRIBUTORS

@@ -1,44 +1,59 @@
-   902	Aldo Cortesi
+  1067	Aldo Cortesi
-   323	Maximilian Hils
+   542	Maximilian Hils
+    76	Marcelo Glezer
     18	Henrik Nordstrom
     13	Thomas Roth
     12	Pedro Worcel
     11	Stephen Altamirano
+    11	Justus Wingert
+    11	Jim Shaver
     10	András Veres-Szentkirályi
+     9	Legend Tang
      8	Rouli
      8	Jason A. Novak
      7	Alexis Hildebrandt
-     5	Tomaz Muraus
-     5	Brad Peabody
      5	Matthias Urlichs
+     5	Brad Peabody
+     5	Tomaz Muraus
+     5	elitest
      4	root
-     4	Marc Liyanage
      4	Valtteri Virtanen
+     4	Wade 524
      4	Bryan Bishop
+     4	Youhei Sakurai
+     4	Marc Liyanage
      3	Chris Neasbitt
      3	Zack B
-     3	Eli Shvartsman
      3	Kyle Manna
-     2	Michael Frister
+     3	Eli Shvartsman
-     2	Bennett Blodinger
+     2	Choongwoo Han
-     2	Jim Lloyd
      2	Rob Wills
      2	israel
-     2	Jaime Soriano Pastor
-     2	Heikki Hannikainen
      2	Mark E. Haase
+     2	Jaime Soriano Pastor
+     2	Jim Lloyd
+     2	Heikki Hannikainen
+     2	Krzysztof Bielicki
+     2	Bennett Blodinger
+     2	Michael Frister
      2	alts
+     1	Yuangxuan Wang
+     1	capt8bit
      1	davidpshaw
      1	deployable
      1	joebowbeer
      1	meeee
+     1	michaeljau
+     1	peralta
      1	phil plante
-     1	Michael Bisbjerg
+     1	sentient07
+     1	vzvu3k6k
      1	Andy Smith
      1	Dan Wilbraham
      1	David Shaw
      1	Eric Entzel
      1	Felix Wolfsteller
+     1	Gabriel Kirkpatrick
      1	Henrik Nordström
      1	Ivaylo Popov
      1	JC
@@ -46,20 +61,28 @@
      1	James Billingham
      1	Jean Regisser
      1	Kit Randel
-     1	Marcelo Glezer
+     1	Lucas Cimon
      1	Mathieu Mitchell
+     1	Michael Bisbjerg
+     1	Mike C
      1	Mikhail Korobov
+     1	Nick HS
+     1	Nick Raptis
      1	Nicolas Esteves
      1	Oleksandr Sheremet
      1	Paul
      1	Rich Somerfield
      1	Rory McCann
      1	Rune Halvorsen
+     1	Ryo Onodera
      1	Sahn Lam
      1	Seppo Yli-Olli
      1	Sergey Chipiga
+     1	Steve Phillips
      1	Steven Van Acker
+     1	Suyash
+     1	Tarashish Mishra
+     1	Terry Long
      1	Ulrich Petri
      1	Vyacheslav Bakhmutov
-     1	Yuangxuan Wang
+     1	Wade Catron
-     1	capt8bit

README.mkd

@@ -75,6 +75,10 @@ command:
 
 ```$ mitmdump --version```
 
+For convenience, the project includes an
+[autoenv](https://github.com/kennethreitz/autoenv) file
+([.env](https://github.com/mitmproxy/mitmproxy/blob/master/.env)) that
+auto-activates the virtualenv when you cd into the mitmproxy directory.
 
 
 ### Testing

check_coding_style.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+autopep8 -i -r -a -a .
+if [[ -n "$(git status -s)" ]]; then
+  echo "autopep8 yielded the following changes:"
+  git status -s
+  git --no-pager diff
+  exit 1
+fi
+
+autoflake -i -r --remove-all-unused-imports --remove-unused-variables .
+if [[ -n "$(git status -s)" ]]; then
+  echo "autoflake yielded the following changes:"
+  git status -s
+  git --no-pager diff
+  exit 1
+fi
+
+echo "Coding style seems to be ok."
+exit 0

dev

@@ -1,7 +1,8 @@
-#!/bin/sh
+#!/bin/bash
+set -e
 VENV=../venv.mitmproxy
 
-virtualenv $VENV
+python -m virtualenv $VENV
 source $VENV/bin/activate
 pip install --src .. -r requirements.txt
 

dev.bat

@@ -2,8 +2,11 @@
 set VENV=..\venv.mitmproxy
 
 virtualenv %VENV%
+if %errorlevel% neq 0 exit /b %errorlevel%
 call %VENV%\Scripts\activate.bat
+if %errorlevel% neq 0 exit /b %errorlevel%
 pip install --src .. -r requirements.txt
+if %errorlevel% neq 0 exit /b %errorlevel%
 
 echo.
 echo * Created virtualenv environment in %VENV%.

doc-src/02-app.css

@@ -0,0 +1,43 @@
+.masthead {
+  text-align: center;
+  border-bottom: 0;
+}
+.frontpage .talks div {
+  margin-bottom: 10px;
+}
+.nav-sidebar {
+  background-color: #f0f0f0;
+  margin-bottom: 20px;
+}
+.nav-sidebar li {
+  line-height: 1.1;
+}
+.nav-sidebar li > a,
+.nav-sidebar .nav-header {
+  padding-left: 20px;
+}
+.nav-sidebar .nav-header {
+  margin-top: 1em;
+  font-size: 1.2em;
+  font-weight: bold;
+}
+.nav-sidebar .active > a,
+.nav-sidebar .active > a:hover,
+.nav-sidebar .active > a:focus {
+  color: #fff;
+  background-color: #428bca;
+}
+.tablenum {
+  font-weight: bold;
+}
+.nowrap {
+  white-space: nowrap;
+}
+.page-header {
+  margin: 0px 0 22px;
+}
+.page-header h1 {
+  margin-top: 0px;
+}
+
+/*# sourceMappingURL=02-app.css.map */

doc-src/02-docstyle.css

@@ -1,20 +0,0 @@
-body {
-    padding-top: 60px;
-    padding-bottom: 40px;
-}
-
-.tablenum {
-    font-weight: bold;
-} 
-
-.nowrap {
-     white-space: nowrap;
-}
-
-h1 {
-    line-height: 1.1;
-}
-
-.page-header {
-    margin: 0px 0 22px;
-}

doc-src/_layout.html

@@ -1,36 +1,44 @@
-<div class="navbar navbar-fixed-top">
+<!DOCTYPE html>
-  <div class="navbar-inner">
+<html>
+    <head>
+        <title>@!pageTitle!@</title>
+        $!header!$
+    </head>
+    <body>
+        <div class="navbar navbar-default navbar-static-top">
             <div class="container">
-      <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+                <div class="navbar-header">
-        <span class="icon-bar"></span>
+                    <a class="navbar-brand" href="@!urlTo("/index.html")!@">
-        <span class="icon-bar"></span>
+                        <img height="20px" src="@!urlTo("mitmproxy-long.png")!@"/>
-        <span class="icon-bar"></span>
                     </a>
-      <a class="brand" href="@!urlTo(idxpath)!@">mitmproxy $!VERSION!$ docs</a>
+                </div>
-      </div><!--/.nav-collapse -->
+                <div class="navbar-header navbar-right">
+                    <a class="navbar-brand" hre="#">$!VERSION!$ docs</a>
+                </div>
             </div>
         </div>
-</div>
 
-<div class="container">
+
+        <div class="container">
           <div class="row">
-    <div class="span3">
+            <div class="col-md-3">
-      <div class="well sidebar-nav">
                 $!navbar!$
             </div>
-    </div>
+            <div class="col-md-9">
-
-    <div class="span9">
                 <div class="page-header">
                 <h1>@!this.title!@</h1>
                 </div>
                 $!body!$
             </div>
           </div>
+        </div>
 
+        <div class="container">
           <hr>
 
           <footer>
             <p>@!copyright!@</p>
           </footer>
-</div>
+        </div>
+    </body>
+</html>

doc-src/_nav.html

@@ -1,6 +1,7 @@
-<ul class="nav nav-list">
+<ul class="nav nav-sidebar">
     $!nav(idxpath, this, state)!$
     $!nav("install.html", this, state)!$
+    $!nav("certinstall.html", this, state)!$
     $!nav("howmitmproxy.html", this, state)!$
     $!nav("modes.html", this, state)!$
 
@@ -28,23 +29,12 @@
         $!nav("upstreamcerts.html", this, state)!$
 
 
-    <li class="nav-header">Installing Certificates</li>
-        $!nav("ssl.html", this, state)!$
-        $!nav("certinstall/webapp.html", this, state)!$
-        $!nav("certinstall/android.html", this, state)!$
-        $!nav("certinstall/firefox.html", this, state)!$
-        $!nav("certinstall/ios.html", this, state)!$
-        $!nav("certinstall/ios-simulator.html", this, state)!$
-        $!nav("certinstall/java.html", this, state)!$
-        $!nav("certinstall/osx.html", this, state)!$
-        $!nav("certinstall/windows7.html", this, state)!$
-
     <li class="nav-header">Transparent Proxying</li>
         $!nav("transparent.html", this, state)!$
         $!nav("transparent/linux.html", this, state)!$
         $!nav("transparent/osx.html", this, state)!$
 
-    <li class="nav-header">Scripting mitmproxy</li>
+    <li class="nav-header">Scripting</li>
         $!nav("scripting/inlinescripts.html", this, state)!$
         $!nav("scripting/libmproxy.html", this, state)!$
 

doc-src/_websitelayout.html

@@ -1,42 +0,0 @@
-<div class="navbar navbar-fixed-top">
-  <div class="navbar-inner">
-    <div class="container">
-      <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </a>
-      <a class="brand" href="@!urlTo(idxpath)!@">mitmproxy</a>
-      <div class="nav">
-        <ul class="nav">
-          <li $!'class="active"' if this.match("/index.html", True) else ""!$> <a href="@!top!@/index.html">home</a> </li>
-          <li $!'class="active"' if this.under("/doc") else ""!$><a href="@!top!@/doc/index.html">docs</a></li>
-          <li $!'class="active"' if this.under("/about.html") else ""!$><a href="@!top!@/about.html">about</a></li>
-        </ul>
-      </div>
-    </div>
-  </div>
-</div>
-
-<div class="container">
-  <div class="row">
-
-    <div class="span3">
-      <div class="well sidebar-nav">
-            $!navbar!$
-      </div>
-    </div>
-    <div class="span9">
-        <div class="page-header">
-        <h1>@!this.title!@</h1>
-        </div>
-        $!body!$
-    </div>
-  </div>
-
-  <hr>
-
-  <footer>
-    <p>@!copyright!@</p>
-  </footer>
-</div>

doc-src/certinstall.html

@@ -0,0 +1,151 @@
+## On This Page
+
+* [Introduction](#docIntro)
+* [Quick Setup](#docQuick)
+* [Installing the mitmproxy CA certificate manually](#docManual)
+* [More on mitmproxy certificates](#docMore)
+* [CA and cert files](#docCertfiles)
+* [Using a custom certificate](#docCustom)
+* [Using a client side certificate](#docClient)
+* [Using a custom certificate authority](#docCA)
+
+## <a id="docIntro"></a>Introduction
+
+Mitmproxy can decrypt encrypted traffic on the fly, as long as the client
+trusts its built-in certificate authority. Usually this means that the
+mitmproxy CA certificates have to be installed on the client device.
+
+## <a id="docQuick"></a>Quick Setup
+
+By far the easiest way to install the mitmproxy certificates is to use the
+built-in certificate installation app. To do this, just start mitmproxy and
+configure your target device with the correct proxy settings. Now start a
+browser on the device, and visit the magic domain **mitm.it**. You should see
+something like this:
+
+<img src="@!urlTo("certinstall-webapp.png")!@"></img>
+
+Click on the relevant icon, and follow the setup instructions for the platform
+you're on, and you are good to go.
+
+
+## <a id="docManual"></a>Installing the mitmproxy CA certificate manually
+
+Sometimes using the quick install app is not an option - Java or the IOS
+similator spring to mind - or you just need to do it manually for some other
+reason. Below is a list of pointers to manual certificate installation
+documentation for some common platforms:
+
+<table class="table">
+    <tr>
+        <td><a href="https://github.com/ADVTOOLS/ADVTrustStore#how-to-use-advtruststore"</a>iOS Simulator</td>
+        <td><a href="http://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html">Java</a></td>
+    </tr>
+    <tr>
+        <td><a href="http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377">iOS</a></td>
+        <td><a href="http://wiki.cacert.org/FAQ/ImportRootCert#Android_Phones_.26_Tablets">Android/Android Simulator</a></td>
+    </tr>
+
+    <tr>
+        <td><a href="http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7">Windows</a></td>
+        <td><a href="https://support.apple.com/kb/PH7297?locale=en_US">Mac OS X</a></td>
+    </tr>
+    <tr>
+        <td><a href="http://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861">Ubuntu/Debian</a></td>
+        <td><a href="https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox">Firefox</a></td>
+    </tr>
+    <tr>
+        <td><a href="https://code.google.com/p/chromium/wiki/LinuxCertManagement">Chrome on Linux</a></td>
+    </tr>
+
+</table>
+
+## <a id="docMore"></a>More on mitmproxy certificates
+
+The first time __mitmproxy__ or __mitmdump__ is run, the mitmproxy Certificate
+Authority(CA) is created in the config directory (~/.mitmproxy by default).
+This CA is used for on-the-fly generation of dummy certificates for each of the
+SSL sites that your client visits. Since your browser won't trust the
+__mitmproxy__ CA out of the box , you will see an SSL certificate warning every
+time you visit a new SSL domain through __mitmproxy__. When you are testing a
+single site through a browser, just accepting the bogus SSL cert manually is
+not too much trouble, but there are a many circumstances where you will want to
+configure your testing system or browser to trust the __mitmproxy__ CA as a
+signing root authority.
+
+
+## <a id="docCertfiles"></a>CA and cert files
+
+The files created by mitmproxy in the .mitmproxy directory are as follows:
+
+<table class="table">
+    <tr>
+        <td class="nowrap">mitmproxy-ca.pem</td>
+        <td>The private key and certificate in PEM format.</td>
+    </tr>
+    <tr>
+        <td class="nowrap">mitmproxy-ca-cert.pem</td>
+        <td>The certificate in PEM format. Use this to distribute to most
+        non-Windows platforms.</td>
+    </tr>
+    <tr>
+        <td class="nowrap">mitmproxy-ca-cert.p12</td>
+        <td>The certificate in PKCS12 format. For use on Windows.</td>
+    </tr>
+    <tr>
+        <td class="nowrap">mitmproxy-ca-cert.cer</td>
+        <td>Same file as .pem, but with an extension expected by some Android
+        devices.</td>
+    </tr>
+</table>
+
+
+## <a id="docCustom"></a>Using a custom certificate
+
+You can use your own certificate by passing the <kbd>--cert</kbd> option to
+mitmproxy. mitmproxy then uses the provided certificate for interception of the
+specified domains instead of generating a certificate signed by its own CA.
+
+The certificate file is expected to be in the PEM format.  You can include
+intermediary certificates right below your leaf certificate, so that you PEM
+file roughly looks like this:
+
+<pre>
+-----BEGIN PRIVATE KEY-----
+&lt;private key&gt;
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+&lt;cert&gt;
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+&lt;intermediary cert (optional)&gt;
+-----END CERTIFICATE-----
+</pre>
+
+For example, you can generate a certificate in this format using these instructions:
+
+<pre class="terminal">
+$ openssl genrsa -out cert.key 2048
+$ openssl req -new -x509 -key cert.key -out cert.crt
+    (Specify the mitm domain as Common Name, e.g. *.google.com)
+$ cat cert.key cert.crt > cert.pem
+$ mitmproxy --cert=cert.pem
+</pre>
+
+## <a id="docClient"></a>Using a client side certificate
+
+You can use a client certificate by passing the <kbd>--client-certs
+DIRECTORY</kbd> option to mitmproxy.  If you visit example.org, mitmproxy looks
+for a file named example.org.pem in the specified directory and uses this as
+the client cert. The certificate file needs to be in the PEM format and should
+contain both the unencrypted private key and the certificate.
+
+
+## <a id="docCA"></a>Using a custom certificate authority
+
+By default, mitmproxy will use <samp>~/.mitmproxy/mitmproxy-ca.pem</samp> as
+the certificate authority to generate certificates for all domains for which no
+custom certificate is provided (see above). You can use your own certificate
+authority by passing the <kbd>--confdir</kbd> option to mitmproxy. Mitmproxy
+will then look for <samp>mitmproxy-ca.pem</samp> in the specified directory. If
+no such file exists, it will be generated automatically.

doc-src/certinstall/android.html

@@ -1,53 +0,0 @@
-The proxy situation on Android is [an
-embarrasment](http://code.google.com/p/android/issues/detail?id=1273).  It's
-scarcely credible, but Android didn't have a global proxy setting at all until
-quite recently, and it's still not supported on many common Android versions.
-In the meantime the app ecosystem has grown used to life without this basic
-necessity, and many apps merrily ignore it even if it's there. This situation
-is improving, but in many circumstances using [transparent
-mode](@!urlTo("transparent.html")!@) is mandatory for testing Android apps.
-
-We used both an Asus Transformer Prime TF201 (Android 4.0.3) and a Nexus 4
-(Android 4.4.4) in the examples below - your device may differ, but the broad
-process should be similar.  On **emulated devices**, there are some [additional
-quirks](https://github.com/mitmproxy/mitmproxy/issues/204#issuecomment-32837093)
-to consider.
-
-
-## Getting the certificate onto the device
-
-The easiest way to get the certificate to the device is to use [the web
-app](@!urlTo("webapp.html")!@). In the rare cases where the web app doesn't
-work, you will need to get the __mitmproxy-ca-cert.cer__ file into the
-__/sdcard__ folder on the device (/sdcard/Download on older devices). This can
-be accomplished in a number of ways:
-
-- If you have the Android Developer Tools installed, you can use [__adb
-push__](http://developer.android.com/tools/help/adb.html).
-- Using a file transfer program like wget (installed on the Android device) to
-copy the file over.
-- Transfer the file using external media like an SD Card. 
-
-Once we have the certificate on the local disk, we need to import it into the
-list of trusted CAs. Go to Settings -> Security -> Credential Storage,
-and select "Install from storage":
-
-<img src="android-settingssecuritymenu.png"/>
-
-The certificate in /sdcard is automatically located and offered for
-installation. Installing the cert will delete the download file from the local
-disk.
-
-
-## Installing the certificate
-
-You should now see something like this (you may have to explicitly name the
-certificate):
-
-<img src="android-settingssecurityinstallca.png"/>
-
-Click OK, and you should then see the certificate listed in the Trusted
-Credentials store:
-
-<img src="android-settingssecurityuserinstalledca.png"/>
-

doc-src/certinstall/firefox.html

@@ -1,31 +0,0 @@
-## Get the certificate to the browser
-
-The easiest way to get the certificate to the browser is to use [the web
-app](@!urlTo("webapp.html")!@). If this fails, do the following:
-
-
-<ol class="tlist">
-    <li> If needed, copy the ~/.mitmproxy/mitmproxy-ca-cert.pem file to the target. </li>
-
-    <li>Open preferences, click on "Advanced", then select"Certificates":
-        <img src="@!urlTo('firefox3.jpg')!@"/>
-    </li>
-
-    <li> Click "View Certificates", "Import", and select the certificate file: 
-        <img src="@!urlTo('firefox3-import.jpg')!@"/>
-    </li>
-
-</ol>
-
-
-## Installing the certificate
-
-<ol class="tlist">
-    <li>Tick "Trust this CA to identify web sites", and click "Ok":
-        <img src="@!urlTo('firefox3-trust.jpg')!@"/>
-    </li>
-
-    <li> You should now see the mitmproxy certificate listed in the Authorities
-    tab.</li>
-</ol>
-

doc-src/certinstall/index.py

@@ -1,13 +0,0 @@
-from countershape import Page
-
-pages = [
-    Page("webapp.html", "Using the Web App"),
-    Page("firefox.html", "Firefox"),
-    Page("osx.html", "OSX"),
-    Page("windows7.html", "Windows 7"),
-    Page("ios.html", "IOS"),
-    Page("ios-simulator.html", "IOS Simulator"),
-    Page("android.html", "Android"),
-    Page("java.html", "Java"),
-    Page("mitm.it-error.html", "Error: No proxy configured"),
-]

doc-src/certinstall/ios-simulator.html

@@ -1,23 +0,0 @@
-
-How to install the __mitmproxy__ certificate authority in the IOS simulator:
-
-<ol class="tlist">
-
-    <li> First, check out the <a
-    href="https://github.com/ADVTOOLS/ADVTrustStore">ADVTrustStore</a> tool
-    from github.</li>
-
-    <li> Now, run the following command:
-
-    <pre class="terminal">./iosCertTrustManager.py -a ~/.mitmproxy/mitmproxy-ca-cert.pem</pre>
-
-    </li>
-
-</ol>
-
-
-Note that although the IOS simulator has its own certificate store, it shares
-the proxy settings of the host operating system. You will therefore to have
-configure your OSX host's proxy settings to use the mitmproxy instance you want
-to test with.
-

doc-src/certinstall/ios.html

@@ -1,27 +0,0 @@
-
-## Getting the certificate onto the device
-
-The easiest way to get the certificate to the device is to use [the web
-app](@!urlTo("webapp.html")!@). In the rare cases where the web app doesn't
-work, you will need to get the __mitmproxy-ca-cert.pem__ file to the device to
-install it. The easiest way to accomplish this is to set up the Mail app on the
-device, and to email it over as an attachment. Open the email, tap on the
-attachment, then proceed with the install.
-
-
-## Installing the certificate
-
-<ol class="tlist">
-    <li>You will be prompted to install a profile. Click "Install":
-
-    <img src="@!urlTo('ios-profile.png')!@"/></li>
-
-    <li>Accept the warning by clicking "Install" again:
-
-    <img src="@!urlTo('ios-warning.png')!@"/></li>
-
-    <li>The certificate should now be trusted:
-
-    <img src="@!urlTo('ios-installed.png')!@"/></li>
-
-</ol>

doc-src/certinstall/java.html

@@ -1,13 +0,0 @@
-
-You can add the mitmproxy certificates to the Java trust store using
-[keytool](http://docs.oracle.com/javase/6/docs/technotes/tools/solaris/keytool.html).
-On OSX, the required command looks like this:
-
-<pre class="terminal">
-sudo keytool -importcert -alias mitmproxy -storepass "password" \
--keystore /System/Library/Java/Support/CoreDeploy.bundle/Contents/Home/lib/security/cacerts \
--trustcacerts -file ~/.mitmproxy/mitmproxy-ca-cert.pem
-</pre>
-
-Note that your store password will (hopefully) be different from the one above.
-

doc-src/certinstall/mitm.it-error.html

@@ -1,5 +0,0 @@
-**Looks like you wanted to install the mitmproxy CA using the web app?**
-
-Unfortunately, there's been no mitmproxy instance on the wire that could have intercepted your request.
-Please configure your client to use mitmproxy and try again.<br>
-The request to <a href="http://mitm.it/">http://mitm.it/</a> must go through your mitmproxy instance.

doc-src/certinstall/osx.html

@@ -1,16 +0,0 @@
-
-How to install the __mitmproxy__ certificate authority in OSX:
-
-<ol class="tlist">
-
-    <li>Open Finder, and double-click on the mitmproxy-ca-cert.pem file.</li>
-
-    <li>You will be prompted to add the certificate. Click "Always Trust": 
-
-        <img src="@!urlTo('osx-addcert-alwaystrust.png')!@"/>
-    </li>
-
-    <li> You may be prompted for your password. You should now see the
-    mitmproxy cert listed under "Certificates".</li>
-</ol>
-

doc-src/certinstall/webapp.html

@@ -1,13 +0,0 @@
-
-By far the easiest way to install the mitmproxy certs is to use the built-in
-web app. To do this, start mitmproxy and configure your target device with the
-correct proxy settings. Now start a browser on the device, and visit the magic
-domain **mitm.it**. You should see something like this:
-
-<img src="@!urlTo("webapp.png")!@"></img>
-
-Just click on the relevant icon, and then follow the setup instructions
-for the platform you're on.
-
-Make sure you aren't using a bandwith optimizer (like Google's Data Compression
-Proxy on Chrome for Android) or the page will not load.

doc-src/certinstall/windows7.html

@@ -1,35 +0,0 @@
-
-How to install the __mitmproxy__ certificate authority in Windows 7:
-
-<ol class="tlist">
-
-    <li> The easiest way to get the certificate to the device is to use <a
-        href="@!urlTo("webapp.html")!@">the web app</a>. If this fails for some
-    reason, simply copy the ~/.mitmproxy/mitmproxy-ca-cert.p12 file to the
-    target system and double-click it.  </li>
-
-    <li> 
-        You should see a certificate import wizard: 
-
-        <img src="@!urlTo('win7-wizard.png')!@"/>
-    </li>
-
-    <li> 
-        Click "Next" until you're prompted for the certificate store:
-
-        <img src="@!urlTo('win7-certstore.png')!@"/>
-
-    </li>
-
-
-    <li> 
-        <p>Select "Place all certificates in the following store", and select "Trusted Root Certification Authorities":</p>
-
-        <img src="@!urlTo('win7-certstore-trustedroot.png')!@"/>
-
-    </li>
-
-    <li> Click "Next" and "Finish". </li>
-
-</ol>
-

doc-src/dev/architecture.html

@@ -1,7 +1,7 @@
-To give you a better understanding of how mitmproxy works, mitmproxy's high-level architecture is detailed
+To give you a better understanding of how mitmproxy works, mitmproxy's
-in the following graphic:
+high-level architecture is detailed in the following graphic:
 
-<img src="@!urlTo('schematics/architecture.png')!@">
+<img class="img-responsive" src="@!urlTo('schematics/architecture.png')!@">
 
 <a href="@!urlTo('schematics/architecture.pdf')!@">(architecture.pdf)</a>
 <p>Please don't refrain from asking any further

doc-src/dev/index.py

@@ -4,5 +4,5 @@ pages = [
     Page("testing.html", "Testing"),
     Page("architecture.html", "Architecture"),
     Page("sslkeylogfile.html", "TLS Master Secrets"),
-#    Page("addingviews.html", "Writing Content Views"),
+    #    Page("addingviews.html", "Writing Content Views"),
 ]

doc-src/features/passthrough.html

@@ -51,10 +51,10 @@ $ mitmdump -v
 127.0.0.1:50588: request
   -> CONNECT example.com:443 HTTP/1.1
 127.0.0.1:50588: Set new server address: example.com:443
-<span style="color: white">127.0.0.1:50588: serverconnect
+127.0.0.1:50588: serverconnect
-  -> example.com:443</span>
+  -> example.com:443
 ^C
-$ <span style="color: white">mitmproxy --ignore ^example\.com:443$</span>    
+$ mitmproxy --ignore ^example\.com:443$
 </pre>
 
 Here are some other examples for ignore patterns:
@@ -62,13 +62,13 @@ Here are some other examples for ignore patterns:
 # Exempt traffic from the iOS App Store (the regex is lax, but usually just works):
 --ignore apple.com:443  
 # "Correct" version without false-positives:
---ignore ^(.+\.)?apple\.com:443$ 
+--ignore '^(.+\.)?apple\.com:443$'
     
 # Ignore example.com, but not its subdomains:
---ignore ^example.com:
+--ignore '^example.com:'
 
 # Ignore everything but example.com and mitmproxy.org:
---ignore ^(?!example\.com)(?!mitmproxy\.org)
+--ignore '^(?!example\.com)(?!mitmproxy\.org)'
     
 # Transparent mode:
 --ignore 17\.178\.96\.59:443

doc-src/howmitmproxy.html

@@ -26,7 +26,7 @@ This is a proxy GET request - an extended form of the vanilla HTTP GET request
 that includes a schema and host specification, and it includes all the
 information mitmproxy needs to proceed.
 
-<img src="explicit.png"/>
+<img class="img-responsive" src="explicit.png"/>
 
 <table class="table">
     <tbody>
@@ -84,7 +84,7 @@ attempts to MITM an SSL connection for analysis. Our answer to this conundrum
 is to become a trusted Certificate Authority ourselves. Mitmproxy includes a
 full CA implementation that generates interception certificates on the fly. To
 get the client to trust these certificates, we [register mitmproxy as a trusted
-CA with the device manually](@!urlTo("ssl.html")!@).
+CA with the device manually](@!urlTo("certinstall.html")!@).
 
 ## Complication 1: What's the remote hostname?
 
@@ -158,7 +158,7 @@ handshake. Luckily, this is almost never an issue in practice.
 
 Lets put all of this together into the complete explicitly proxied HTTPS flow.
 
-<img src="explicit_https.png"/>
+<img class="img-responsive" src="explicit_https.png"/>
 
 <table class="table">
     <tbody>
@@ -250,7 +250,7 @@ mitmproxy, this takes the form of a built-in set of
 that know how to talk to each platform's redirection mechanism.  Once we have
 this information, the process is fairly straight-forward.
 
-<img src="transparent.png"/>
+<img class="img-responsive" src="transparent.png"/>
 
 
 <table class="table">
@@ -296,7 +296,7 @@ transparently proxying HTTP, and explicitly proxying HTTPS. We use the routing
 mechanism to establish the upstream server address, and then proceed as for
 explicit HTTPS connections to establish the CN and SANs, and cope with SNI.
 
-<img src="transparent_https.png"/>
+<img class="img-responsive" src="transparent_https.png"/>
 
 
 <table class="table">

doc-src/index.py

@@ -2,7 +2,7 @@ import os
 import sys
 import datetime
 import countershape
-from countershape import Page, Directory, markup, model
+from countershape import Page, Directory, markup
 import countershape.template
 
 MITMPROXY_SRC = os.path.abspath(
@@ -15,10 +15,10 @@ ns.VERSION = version.VERSION
 
 if ns.options.website:
     ns.idxpath = "doc/index.html"
-    this.layout = countershape.Layout("_websitelayout.html")
 else:
     ns.idxpath = "index.html"
-    this.layout = countershape.Layout("_layout.html")
+
+this.layout = countershape.layout.FileLayout("_layout.html")
 
 ns.title = countershape.template.Template(None, "<h1>@!this.title!@</h1>")
 this.titlePrefix = "%s - " % version.NAMEVERSION
@@ -37,7 +37,7 @@ def mpath(p):
 def example(s):
     d = file(mpath(s)).read().rstrip()
     extemp = """<div class="example">%s<div class="example_legend">(%s)</div></div>"""
-    return extemp%(countershape.template.Syntax("py")(d), s)
+    return extemp % (countershape.template.Syntax("py")(d), s)
 
 
 ns.example = example
@@ -52,7 +52,8 @@ def nav(page, current, state):
     else:
         pre = "<li>"
     p = state.application.getPage(page)
-    return pre + '<a href="%s">%s</a></li>'%(model.UrlTo(page), p.title)
+    return pre + \
+        '<a href="%s">%s</a></li>' % (countershape.widgets.UrlTo(page), p.title)
 ns.nav = nav
 ns.navbar = countershape.template.File(None, "_nav.html")
 
@@ -60,6 +61,7 @@ ns.navbar = countershape.template.File(None, "_nav.html")
 pages = [
     Page("index.html", "Introduction"),
     Page("install.html", "Installation"),
+    Page("certinstall.html", "About Certificates"),
     Page("howmitmproxy.html", "How mitmproxy works"),
     Page("modes.html", "Modes of Operation"),
 
@@ -67,8 +69,6 @@ pages = [
     Page("mitmdump.html", "mitmdump"),
     Page("config.html", "configuration"),
 
-    Page("ssl.html", "Overview"),
-    Directory("certinstall"),
     Directory("scripting"),
     Directory("tutorials"),
     Page("transparent.html", "Overview"),

doc-src/install.html

@@ -1,40 +1,33 @@
+## On This Page
 
-## Installing from source
+* [Installation On Ubuntu](#docUbuntu)
+* [Installation On Mac OS X](#docOSX)
+* [Installation On Windows](#docWindows)
 
-The preferred way to install mitmproxy - whether you're installing the latest
+## <a id=docUbuntu></a>Installation On Ubuntu
-release or from source - is to use [pip](http://www.pip-installer.org/). If you
+
-don't already have pip on your system, you can find installation instructions
+Ubuntu comes with Python but we need to install pip, python-dev and several libraries.  This was tested on a fully patched installation of Ubuntu 14.04.
-[here](http://www.pip-installer.org/en/latest/installing.html).
 
 <pre class="terminal">
-pip install mitmproxy
+$ sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev
+
+$ sudo pip install mitmproxy
 </pre>
 
-If you also want to install the optional packages AMF, protobuf and CSS
+Once installation is complete you can run <a href="mitmproxy.html">mitmproxy</a> or <a href="mitmdump.html">mitmdump</a> from a terminal.
-content views, do this:
 
-<pre class="terminal">
+### Installation From Source
-pip install "mitmproxy[contentviews]"
+
-</pre>
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to get set up to contribute to the project,
+install the dependencies as you would for a regular mitmproxy installation (see previous section).
+Then see the <a href="https://github.com/mitmproxy/mitmproxy/blob/master/README.mkd#hacking">Hacking</a> section of the README on GitHub.
 
 
-## OSX
+## <a id=docOSX></a>Installation On Mac OS X
 
-The easiest way to get up and running on OSX is to download the pre-built
+The easiest way to get up and running on OSX is to download the pre-built binary packages from [mitmproxy.org](http://mitmproxy.org).
-binary packages from [mitmproxy.org](http://mitmproxy.org). If you still want
-to install using pip, there are a few things to keep in mind:
 
-- If you're running a Python interpreter installed with homebrew (or similar),
+There are a few bits of customization you might want to do to make mitmproxy comfortable to use on OSX. The default color scheme is optimized for a dark background terminal, but you can select a palette for a light terminal background with the --palette option. You can use the OSX <b>open</b> program to create a simple and effective <b>~/.mailcap</b> file to view request and response bodies:
-you may have to install some dependencies by hand. 
-- Make sure that XCode is installed from the App Store, and that the
-command-line tools have been downloaded (XCode/Preferences/Downloads).
-
-There are a few bits of customization you might want to do to make mitmproxy
-comfortable to use on OSX. The default color scheme is optimized for a dark
-background terminal, but you can select a palette for a light terminal
-background with the --palette option. You can use the OSX <b>open</b> program
-to create a simple and effective <b>~/.mailcap</b> file to view request and
-response bodies:
 
 <pre class="terminal">
 application/*; /usr/bin/open -Wn %s
@@ -43,17 +36,44 @@ image/*; /usr/bin/open -Wn %s
 video/*; /usr/bin/open -Wn %s
 </pre>
 
+Once installation is complete you can run <a href="mitmproxy.html">mitmproxy</a> or <a href="mitmdump.html">mitmdump</a> from a terminal.
 
-## Ubuntu
+### Installation From Source
 
-On Ubuntu, you will need the following native packages to install mitmproxy
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to get set up to contribute to the project, ithere are a few OS X specific things to keep in mind.
-from source:
 
-- build-essential
+- Make sure that XCode is installed from the App Store, and that the command-line tools have been downloaded (XCode/Preferences/Downloads).
-- python-dev
+- If you're running a Python interpreter installed with homebrew (or similar), you may have to install some dependencies by hand. 
-- libffi-dev
+
-- libssl-dev
+Then see the <a href="https://github.com/mitmproxy/mitmproxy/blob/master/README.mkd#hacking">Hacking</a> section of the README on GitHub.
-- libxml2-dev
+
-- libxslt1-dev
+## <a id=docWindows></a>Installation On Windows
+
+Please note that mitmdump is the only component of mitmproxy that is supported on Windows at the moment.
+There is no interactive user interface on Windows.
+
+
+First, install the latest version of Python 2.7 from the <a href="https://www.python.org/downloads/windows/">Python website</a>.
+If you already have an older version of Python 2.7 installed, make sure to install <a href="https://pip.pypa.io/en/latest/installing.html">pip</a>
+(pip is included in Python 2.7.9+ by default).
+
+Next, add Python and the Python Scripts directory to your <strong>PATH</strong> variable. You can do this easily by running the following in powershell:
+
+<pre class="terminal">
+[Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27\;C:\Python27\Scripts\", "User")
+</pre>
+
+
+Now, you can install mitmproxy by running
+
+<pre class="terminal">
+pip install mitmproxy
+</pre>
+
+Once the installation is complete, you can run <a href="mitmdump.html">mitmdump</a> from a command prompt.
+
+### Installation From Source
+
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to get set up to contribute to the project, install Python as outlined above, then see the <a href="https://github.com/mitmproxy/mitmproxy/blob/master/README.mkd#hacking">Hacking</a> section of the README on GitHub.
 
 

doc-src/mitmproxy.html

@@ -9,7 +9,7 @@ documentation from any __mitmproxy__ screen.
 
 The flow list shows an index of captured flows in chronological order.
 
-<img src="@!urlTo('screenshots/mitmproxy.png')!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy.png')!@"/>
 
 - __1__: A GET request, returning a 302 Redirect response.
 - __2__: A GET request, returning 16.75kb of text/html data.
@@ -32,7 +32,7 @@ interfaces.
 
 The __Flow View__ lets you inspect and manipulate a single flow:
 
-<img src="@!urlTo('screenshots/mitmproxy-flowview.png')!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-flowview.png')!@"/>
 
 - __1__: Flow summary.
 - __2__: The Request/Response tabs, showing you which part of the flow you are
@@ -65,13 +65,13 @@ At the moment, the Grid Editor is used in four parts of mitmproxy:
 If there is is no data, an empty editor will be started to let you add some.
 Here is the editor showing the headers from a request:
 
-<img src="@!urlTo('screenshots/mitmproxy-kveditor.png')!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-kveditor.png')!@"/>
 
 To edit, navigate to the key or value you want to modify using the arrow or vi
 navigation keys, and press enter. The background color will change to show that
 you are in edit mode for the specified field:
 
-<img src="@!urlTo('screenshots/mitmproxy-kveditor-editmode.png')!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-kveditor-editmode.png')!@"/>
 
 Modify the field as desired, then press escape to exit edit mode when you're
 done. You can also add a row (_a_ key), delete a row (_d_ key), spawn an
@@ -88,7 +88,7 @@ or client.
 
 ### 1: Set an interception pattern
 
-<img src="@!urlTo('mitmproxy-intercept-filt.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-filt.png')!@"/>
 
 We press _i_ to set an interception pattern. In this case, the __~q__ filter
 pattern tells __mitmproxy__ to intercept all requests. For complete filter
@@ -97,18 +97,18 @@ document, or the built-in help function in __mitmproxy__.
 
 ### 2: Intercepted connections are indicated with orange text:
 
-<img src="@!urlTo('mitmproxy-intercept-mid.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-mid.png')!@"/>
 
 ### 3: You can now view and modify the request:
 
-<img src="@!urlTo('mitmproxy-intercept-options.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-options.png')!@"/>
 
 In this case, we viewed the request by selecting it, pressed _e_ for "edit"
 and _m_ for "method" to change the HTTP request method.
 
 ### 4: Accept the intercept to continue:
 
-<img src="@!urlTo('mitmproxy-intercept-result.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-result.png')!@"/>
 
 Finally, we press _a_ to accept the modified request, which is then sent on to
 the server. In this case, we changed the request from an HTTP GET to

doc-src/modes.html

@@ -9,7 +9,7 @@ variety of scenarios:
 
 Now, which one should you pick? Use this flow chart:
 
-<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
+<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
 
 <div class="page-header">
     <h1>Regular Proxy</h1>
@@ -31,7 +31,7 @@ these cases, you need to use mitmproxy's transparent mode.
 
 If you are proxying an external device, your network will probably look like this:
 
-<img src="@!urlTo('schematics/proxy-modes-regular.png')!@">
+<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-regular.png')!@">
 
 The square brackets signify the source and destination IP addresses. Your
 client explicitly connects to mitmproxy and mitmproxy explicitly connects
@@ -48,7 +48,7 @@ below, a machine running mitmproxy has been inserted between the router and
 the internet:
 
 <a href="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
 </a>
 
 The square brackets signify the source and destination IP addresses. Round
@@ -60,7 +60,7 @@ remove the target information, leaving mitmproxy unable to determine the real
 destination.
 
 <a href="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
 
 <h2>Common Configurations</h2>
 
@@ -79,7 +79,7 @@ intact, is to simply configure the client with the mitmproxy box as the
 default gateway.
 
 <a href="@!urlTo('schematics/proxy-modes-transparent-2.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
 
 In this scenario, we would:
 
@@ -141,7 +141,7 @@ packet filter you're using. In most cases, the configuration will look like
 this:
 
 <a href="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
 </a>
 
 
@@ -154,7 +154,7 @@ Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal
 HTTP server:
 
 <a href="@!urlTo('schematics/proxy-modes-reverse.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
 </a>
 
 There are various use-cases:
@@ -215,7 +215,7 @@ appliance, you can use mitmproxy's upstream mode. In upstream mode, all
 requests are unconditionally transferred to an upstream proxy of your choice.
 
 <a href="@!urlTo('schematics/proxy-modes-upstream.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
 
 mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy
 mode. You could in theory chain multiple mitmproxy instances in a row, but

doc-src/scripting/inlinescripts.html

@@ -122,12 +122,11 @@ The main classes you will deal with in writing mitmproxy scripts are:
         <td> A handle for interacting with mitmproxy's from within scripts.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.ODict</th>
+        <th>netlib.odict.ODict</th>
 
         <td>A dictionary-like object for managing sets of key/value data. There
-            is also a variant called CaselessODict that ignores key case for some
+        is also a variant called ODictCaseless that ignores key case for some
-            calls (used mainly for headers).
+        calls (used mainly for headers).</td>
-        </td>
     </tr>
     <tr>
         <th>netlib.certutils.SSLCert</th>

doc-src/ssl.html

@@ -1,99 +0,0 @@
-
-The first time __mitmproxy__ or __mitmdump__ is run, a set of certificate files
-for the mitmproxy Certificate Authority are created in the config directory
-(~/.mitmproxy by default). This CA is used for on-the-fly generation of dummy
-certificates for SSL interception. Since your browser won't trust the
-__mitmproxy__ CA out of the box (and rightly so), you will see an SSL cert
-warning every time you visit a new SSL domain through __mitmproxy__. When
-you're testing a single site through a browser, just accepting the bogus SSL
-cert manually is not too much trouble, but there are a many circumstances where
-you will want to configure your testing system or browser to trust the
-__mitmproxy__ CA as a signing root authority.
-
-
-CA and cert files
------------------
-
-The files created by mitmproxy in the .mitmproxy directory are as follows: 
-
-<table class="table">
-    <tr>
-        <td class="nowrap">mitmproxy-ca.pem</td>
-        <td>The private key and certificate in PEM format.</td>
-    </tr>
-    <tr>
-        <td class="nowrap">mitmproxy-ca-cert.pem</td>
-        <td>The certificate in PEM format. Use this to distribute to most
-        non-Windows platforms.</td>
-    </tr>
-    <tr>
-        <td class="nowrap">mitmproxy-ca-cert.p12</td>
-        <td>The certificate in PKCS12 format. For use on Windows.</td>
-    </tr>
-    <tr>
-        <td class="nowrap">mitmproxy-ca-cert.cer</td>
-        <td>Same file as .pem, but with an extension expected by some Android
-        devices.</td>
-    </tr>
-</table>
-    
-
-Using a custom certificate
---------------------------
-
-You can use your own certificate by passing the <kbd>--cert</kbd> option to mitmproxy. mitmproxy then uses the provided
-certificate for interception of the specified domains instead of generating a cert signed by its own CA.
-
-The certificate file is expected to be in the PEM format.
-You can include intermediary certificates right below your leaf certificate, so that you PEM file roughly looks like
-this:
-
-<pre>
------BEGIN PRIVATE KEY-----
-&lt;private key&gt;
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-&lt;cert&gt;
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-&lt;intermediary cert (optional)&gt;
------END CERTIFICATE-----
-</pre>
-
-For example, you can generate a certificate in this format using these instructions:
-
-<pre class="terminal">
-> openssl genrsa -out cert.key 8192
-> openssl req -new -x509 -key cert.key -out cert.crt
-    (Specify the mitm domain as Common Name, e.g. *.google.com)
-> cat cert.key cert.crt > cert.pem
-> mitmproxy --cert=cert.pem
-</pre>
-
-Using a client side certificate
-------------------------------------
-You can use a client certificate by passing the <kbd>--client-certs DIRECTORY</kbd> option to mitmproxy.
-If you visit example.org, mitmproxy looks for a file named example.org.pem in the specified directory
-and uses this as the client cert. The certificate file needs to be in the PEM format and should contain
-both the unencrypted private key as well as the certificate.
-
-
-Using a custom certificate authority
-------------------------------------
-
-By default, mitmproxy will (generate and) use <samp>~/.mitmproxy/mitmproxy-ca.pem</samp> as the default certificate
-authority to generate certificates for all domains for which no custom certificate is provided (see above).
-You can use your own certificate authority by passing the <kbd>--confdir</kbd> option to mitmproxy.
-mitmproxy will then look for <samp>mitmproxy-ca.pem</samp> in the specified directory. If no such file exists,
-it will be generated automatically.
-
-Installing the mitmproxy CA
----------------------------
-
-* [Firefox](@!urlTo("certinstall/firefox.html")!@)
-* [OSX](@!urlTo("certinstall/osx.html")!@)
-* [Windows 7](@!urlTo("certinstall/windows7.html")!@)
-* [iPhone/iPad](@!urlTo("certinstall/ios.html")!@)
-* [IOS Simulator](@!urlTo("certinstall/ios-simulator.html")!@)
-* [Android](@!urlTo("certinstall/android.html")!@)
-

doc-src/transparent/linux.html

@@ -3,7 +3,7 @@ achieve transparent mode.
 
 <ol class="tlist">
 
-    <li> <a href="@!urlTo('ssl.html')!@">Install the mitmproxy
+    <li> <a href="@!urlTo('certinstall.html')!@">Install the mitmproxy
     certificates on the test device</a>. </li>
 
     <li> Enable IP forwarding:

doc-src/transparent/osx.html

@@ -7,7 +7,7 @@ OSX.
 
 <ol class="tlist">
 
-    <li> <a href="@!urlTo('ssl.html')!@">Install the mitmproxy
+    <li> <a href="@!urlTo('certinstall.html')!@">Install the mitmproxy
     certificates on the test device</a>. </li>
 
     <li> Enable IP forwarding:
@@ -76,6 +76,3 @@ want to intercept your OSX traffic, you should use an external host to run
 mitmproxy. None the less, pf is flexible to cater for a range of creative
 possibilities, like intercepting traffic emanating from VMs.  See the
 **pf.conf** man page for more.
-
-
-

doc-src/tutorials/gamecenter.html

@@ -2,11 +2,9 @@
 ## The setup
 
 In this tutorial, I'm going to show you how simple it is to creatively
-interfere with Apple Game Center traffic using mitmproxy. To set things up, I
+interfere with Apple Game Center traffic using mitmproxy. To set things up,
-registered my mitmproxy CA certificate with my iPhone - there's a [step by step
+install the [mitmproxy root certificate](@!urlTo("certinstall.html")!@). Then
-set of instructions](@!urlTo("certinstall/ios.html")!@) elsewhere in this manual. I then
+start mitmproxy on your desktop, and confige the iPhone to use it as a proxy.
-started mitmproxy on my desktop, and configured the iPhone to use it as a
-proxy. 
 
 
 ## Taking a look at the Game Center traffic
@@ -17,14 +15,14 @@ Worm](http://itunes.apple.com/us/app/super-mega-worm/id388541990?mt=8) - a
 great little retro-apocalyptic sidescroller for the iPhone:
 
 <center>
-    <img src="@!urlTo('tutorials/supermega.png')!@"/>
+    <img class="img-responsive" src="@!urlTo('tutorials/supermega.png')!@"/>
 </center>
 
 After finishing a game (take your time), watch the traffic flowing through
 mitmproxy:
 
 <center>
-    <img src="@!urlTo('tutorials/one.png')!@"/>
+    <img class="img-responsive" src="@!urlTo('tutorials/one.png')!@"/>
 </center>
 
 We see a bunch of things we might expect - initialisation, the retrieval of
@@ -99,7 +97,7 @@ replay.
 ## The glorious result and some intrigue
 
 <center>
-    <img src="@!urlTo('tutorials/leaderboard.png')!@"/>
+    <img class="img-responsive" src="@!urlTo('tutorials/leaderboard.png')!@"/>
 </center>
 
 And that's it - according to the records, I am the greatest Super Mega Worm
@@ -119,4 +117,3 @@ phone, then 2^31-1 might well be the maximum score you could get. Then again,
 if the game itself stores its score in a signed 32-bit int, you could get the
 same score through perfect play, effectively beating the game. So, which is it
 in this case? I'll leave that for you to decide.
-

doc-src/tutorials/transparent-dhcp.html

@@ -4,10 +4,10 @@ This walkthrough illustrates how to set up transparent proxying with mitmproxy.
     The network setup is simple:  `internet <--> proxy vm <--> (virtual) internal network`.
     For the proxy machine, *eth0* represents the outgoing network. *eth1* is connected to the internal network that will be proxified, using a static ip (192.168.3.1).
     <hr>VirtualBox configuration:
-    <img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth0.png')!@"/><br><br>
+    <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth0.png')!@"/><br><br>
-    <img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth1.png')!@"/>  
+    <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth1.png')!@"/>
     <br>Proxy VM:
-    <img src="@!urlTo('tutorials/transparent-dhcp/step1_proxy.png')!@"/>  
+    <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_proxy.png')!@"/>
     <hr>
 2.  **Configure DHCP and DNS**
     We use dnsmasq to provide DHCP and DNS in our internal network.
@@ -34,7 +34,7 @@ This walkthrough illustrates how to set up transparent proxying with mitmproxy.
         `sudo service dnsmasq restart`
         <hr>
         Your proxied machine's network settings should now look similar to this:
-        <img src="@!urlTo('tutorials/transparent-dhcp/step2_proxied_vm.png')!@"/>
+        <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step2_proxied_vm.png')!@"/>
         <hr>
 
 3.  **Set up traffic redirection to mitmproxy**
@@ -46,9 +46,8 @@ This walkthrough illustrates how to set up transparent proxying with mitmproxy.
         -j REDIRECT --to-port 8080
     </pre>
 
-4.  If required, <a href="@!urlTo('ssl.html')!@">install the mitmproxy
+4.  If required, <a href="@!urlTo('certinstall.html')!@">install the mitmproxy
     certificates on the test device</a>.
 
 5.  Finally, we can run <code>mitmproxy -T</code>.
     The proxied machine cannot to leak any data outside of HTTP or DNS requests.
-

examples/README

@@ -3,6 +3,8 @@ add_header.py             Simple script that just adds a header to every request
 change_upstream_proxy.py  Dynamically change the upstream proxy
 dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
 dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
+filt.py                   Use mitmproxy's filter expressions in your script.
+flowwriter.py             Only write selected flows into a mitmproxy dumpfile.
 iframe_injector.py        Inject configurable iframe into pages.
 modify_form.py            Modify all form submissions to add a parameter.
 modify_querystring.py     Modify all query strings to add a parameters.

examples/change_upstream_proxy.py

@@ -1,10 +1,13 @@
 # This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy
 # in upstream proxy mode.
 #
-# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s "change_upstream_proxy.py host"
+# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s
+# "change_upstream_proxy.py host"
 from libmproxy.protocol.http import send_connect_request
 
 alternative_upstream_proxy = ("localhost", 8082)
+
+
 def should_redirect(flow):
     return flow.request.host == "example.com"
 
@@ -15,7 +18,12 @@ def request(context, flow):
         # If you want to change the target server, you should modify flow.request.host and flow.request.port
         # flow.live.change_server should only be used by inline scripts to change the upstream proxy,
         # unless you are sure that you know what you are doing.
-        server_changed = flow.live.change_server(alternative_upstream_proxy, persistent_change=True)
+        server_changed = flow.live.change_server(
+            alternative_upstream_proxy,
+            persistent_change=True)
         if flow.request.scheme == "https" and server_changed:
-            send_connect_request(flow.live.c.server_conn, flow.request.host, flow.request.port)
+            send_connect_request(
+                flow.live.c.server_conn,
+                flow.request.host,
+                flow.request.port)
             flow.live.c.establish_ssl(server=True)

examples/dns_spoofing.py

@@ -25,9 +25,11 @@ mitmproxy -p 443 -R https2http://localhost:8000
 def request(context, flow):
     if flow.client_conn.ssl_established:
         # TLS SNI or Host header
-        flow.request.host = flow.client_conn.connection.get_servername() or flow.request.pretty_host(hostheader=True)
+        flow.request.host = flow.client_conn.connection.get_servername(
+        ) or flow.request.pretty_host(hostheader=True)
 
-        # If you use a https2http location as default destination, these attributes need to be corrected as well:
+        # If you use a https2http location as default destination, these
+        # attributes need to be corrected as well:
         flow.request.port = 443
         flow.request.scheme = "https"
     else:

examples/filt.py

@@ -0,0 +1,16 @@
+# This scripts demonstrates how to use mitmproxy's filter pattern in inline scripts.
+# Usage: mitmdump -s "filt.py FILTER"
+
+from libmproxy import filt
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError("Usage: -s 'filt.py FILTER'")
+    context.filter = filt.parse(argv[1])
+
+
+def response(context, flow):
+    if flow.match(context.filter):
+        print("Flow matches filter:")
+        print(flow)

examples/flowbasic

@@ -36,7 +36,8 @@ class MyMaster(flow.FlowMaster):
 
 config = proxy.ProxyConfig(
     port=8080,
-    cadir="~/.mitmproxy/"  # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
+    # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
+    cadir="~/.mitmproxy/"
 )
 state = flow.State()
 server = ProxyServer(config)

examples/flowwriter.py

@@ -0,0 +1,20 @@
+import random
+import sys
+
+from libmproxy.flow import FlowWriter
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError('Usage: -s "flowriter.py filename"')
+
+    if argv[1] == "-":
+        f = sys.stdout
+    else:
+        f = open(argv[1], "wb")
+    context.flow_writer = FlowWriter(f)
+
+
+def response(context, flow):
+    if random.choice([True, False]):
+        context.flow_writer.add(flow)

examples/har_extractor.py

@@ -83,7 +83,8 @@ def response(context, flow):
         # Calculate the connect_time for this server_conn. Afterwards add it to
         # seen list, in order to avoid the connect_time being present in entries
         # that use an existing connection.
-        connect_time = flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start
+        connect_time = flow.server_conn.timestamp_tcp_setup - \
+            flow.server_conn.timestamp_start
         context.seen_server.add(flow.server_conn)
 
         if flow.server_conn.timestamp_ssl_setup is not None:
@@ -91,7 +92,8 @@ def response(context, flow):
             # the start of the successful tcp setup and the successful ssl
             # setup. If  no ssl setup has been made it is left as -1 since it
             # doesn't apply to this connection.
-            ssl_time = flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup
+            ssl_time = flow.server_conn.timestamp_ssl_setup - \
+                flow.server_conn.timestamp_tcp_setup
 
     # Calculate the raw timings from the different timestamps present in the
     # request and response object. For lack of a way to measure it dns timings
@@ -110,7 +112,8 @@ def response(context, flow):
 
     # HAR timings are integers in ms, so we have to re-encode the raw timings to
     # that format.
-    timings = dict([(key, int(1000 * value)) for key, value in timings_raw.iteritems()])
+    timings = dict([(key, int(1000 * value))
+                    for key, value in timings_raw.iteritems()])
 
     # The full_time is the sum of all timings. Timings set to -1 will be ignored
     # as per spec.
@@ -119,20 +122,27 @@ def response(context, flow):
         if item > -1:
             full_time += item
 
-    started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, tz=utc).isoformat()
+    started_date_time = datetime.fromtimestamp(
+        flow.request.timestamp_start,
+        tz=utc).isoformat()
 
-    request_query_string = [{"name": k, "value": v} for k, v in flow.request.get_query()]
+    request_query_string = [{"name": k, "value": v}
+                            for k, v in flow.request.get_query()]
     request_http_version = ".".join([str(v) for v in flow.request.httpversion])
     # Cookies are shaped as tuples by MITMProxy.
-    request_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.request.get_cookies() or {}).iteritems()]
+    request_cookies = [{"name": k.strip(), "value": v[0]}
+                       for k, v in (flow.request.get_cookies() or {}).iteritems()]
     request_headers = [{"name": k, "value": v} for k, v in flow.request.headers]
     request_headers_size = len(str(flow.request.headers))
     request_body_size = len(flow.request.content)
 
-    response_http_version = ".".join([str(v) for v in flow.response.httpversion])
+    response_http_version = ".".join(
+        [str(v) for v in flow.response.httpversion])
     # Cookies are shaped as tuples by MITMProxy.
-    response_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.response.get_cookies() or {}).iteritems()]
+    response_cookies = [{"name": k.strip(), "value": v[0]}
-    response_headers = [{"name": k, "value": v} for k, v in flow.response.headers]
+                        for k, v in (flow.response.get_cookies() or {}).iteritems()]
+    response_headers = [{"name": k, "value": v}
+                        for k, v in flow.response.headers]
     response_headers_size = len(str(flow.response.headers))
     response_body_size = len(flow.response.content)
     response_body_decoded_size = len(flow.response.get_decoded_content())
@@ -140,33 +150,43 @@ def response(context, flow):
     response_mime_type = flow.response.headers.get_first('Content-Type', '')
     response_redirect_url = flow.response.headers.get_first('Location', '')
 
-    entry = HAR.entries({"startedDateTime": started_date_time,
+    entry = HAR.entries(
+        {
+            "startedDateTime": started_date_time,
             "time": full_time,
-                         "request": {"method": flow.request.method,
+            "request": {
+                "method": flow.request.method,
                 "url": flow.request.url,
                 "httpVersion": request_http_version,
                 "cookies": request_cookies,
                 "headers": request_headers,
                 "queryString": request_query_string,
                 "headersSize": request_headers_size,
-                                     "bodySize": request_body_size, },
+                "bodySize": request_body_size,
-                         "response": {"status": flow.response.code,
+            },
+            "response": {
+                "status": flow.response.code,
                 "statusText": flow.response.msg,
                 "httpVersion": response_http_version,
                 "cookies": response_cookies,
                 "headers": response_headers,
-                                      "content": {"size": response_body_size,
+                "content": {
+                    "size": response_body_size,
                     "compression": response_body_compression,
                     "mimeType": response_mime_type},
                 "redirectURL": response_redirect_url,
                 "headersSize": response_headers_size,
-                                      "bodySize": response_body_size, },
+                "bodySize": response_body_size,
+            },
             "cache": {},
-                         "timings": timings, })
+            "timings": timings,
+        })
 
     # If the current url is in the page list of context.HARLog or does not have
     # a referrer we add it as a new pages object.
-    if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get('Referer', None) is None:
+    if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get(
+            'Referer',
+            None) is None:
         page_id = context.HARLog.create_page_id()
         context.HARLog.add(
             HAR.pages({

examples/iframe_injector.py

@@ -16,7 +16,12 @@ def response(context, flow):
     with decoded(flow.response):  # Remove content encoding (gzip, ...)
         html = BeautifulSoup(flow.response.content)
         if html.body:
-            iframe = html.new_tag("iframe", src=context.iframe_url, frameborder=0, height=0, width=0)
+            iframe = html.new_tag(
+                "iframe",
+                src=context.iframe_url,
+                frameborder=0,
+                height=0,
+                width=0)
             html.body.insert(0, iframe)
             flow.response.content = str(html)
             context.log("Iframe inserted.")

examples/ignore_websocket.py

@@ -24,9 +24,11 @@ def done(context):
     HTTPRequest._headers_to_strip_off.append("Connection")
     HTTPRequest._headers_to_strip_off.append("Upgrade")
 
+
 @concurrent
 def response(context, flow):
-    if flow.response.headers.get_first("Connection", None) == "Upgrade":
+    value = flow.response.headers.get_first("Connection", None)
+    if value and value.upper() == "UPGRADE":
         # We need to send the response manually now...
         flow.client_conn.send(flow.response.assemble())
         # ...and then delegate to tcp passthrough.

examples/mitmproxywrapper.py

@@ -14,6 +14,7 @@ import contextlib
 import os
 import sys
 
+
 class Wrapper(object):
 
     def __init__(self, port, extra_arguments=None):
@@ -21,16 +22,21 @@ class Wrapper(object):
         self.extra_arguments = extra_arguments
 
     def run_networksetup_command(self, *arguments):
-        return subprocess.check_output(['sudo', 'networksetup'] + list(arguments))
+        return subprocess.check_output(
+            ['sudo', 'networksetup'] + list(arguments))
 
     def proxy_state_for_service(self, service):
-        state = self.run_networksetup_command('-getwebproxy', service).splitlines()
+        state = self.run_networksetup_command(
+            '-getwebproxy',
+            service).splitlines()
         return dict([re.findall(r'([^:]+): (.*)', line)[0] for line in state])
 
     def enable_proxy_for_service(self, service):
         print('Enabling proxy on {}...'.format(service))
         for subcommand in ['-setwebproxy', '-setsecurewebproxy']:
-            self.run_networksetup_command(subcommand, service, '127.0.0.1', str(self.port))
+            self.run_networksetup_command(
+                subcommand, service, '127.0.0.1', str(
+                    self.port))
 
     def disable_proxy_for_service(self, service):
         print('Disabling proxy on {}...'.format(service))
@@ -39,11 +45,17 @@ class Wrapper(object):
 
     def interface_name_to_service_name_map(self):
         order = self.run_networksetup_command('-listnetworkserviceorder')
-        mapping = re.findall(r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$', order, re.MULTILINE)
+        mapping = re.findall(
+            r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$',
+            order,
+            re.MULTILINE)
         return dict([(b, a) for (a, b) in mapping])
 
     def run_command_with_input(self, command, input):
-        popen = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        popen = subprocess.Popen(
+            command,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE)
         (stdout, stderr) = popen.communicate(input)
         return stdout
 
@@ -54,13 +66,15 @@ class Wrapper(object):
         return interface
 
     def primary_service_name(self):
-        return self.interface_name_to_service_name_map()[self.primary_interace_name()]
+        return self.interface_name_to_service_name_map()[
+            self.primary_interace_name()]
 
     def proxy_enabled_for_service(self, service):
         return self.proxy_state_for_service(service)['Enabled'] == 'Yes'
 
     def toggle_proxy(self):
-        new_state = not self.proxy_enabled_for_service(self.primary_service_name())
+        new_state = not self.proxy_enabled_for_service(
+            self.primary_service_name())
         for service_name in self.connected_service_names():
             if self.proxy_enabled_for_service(service_name) and not new_state:
                 self.disable_proxy_for_service(service_name)
@@ -74,8 +88,11 @@ class Wrapper(object):
 
         service_names = []
         for service_id in service_ids:
-            scutil_script = 'show Setup:/Network/Service/{}\n'.format(service_id)
+            scutil_script = 'show Setup:/Network/Service/{}\n'.format(
-            stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
+                service_id)
+            stdout = self.run_command_with_input(
+                '/usr/sbin/scutil',
+                scutil_script)
             service_name, = re.findall(r'UserDefinedName\s*:\s*(.+)', stdout)
             service_names.append(service_name)
 
@@ -119,11 +136,19 @@ class Wrapper(object):
     def main(cls):
         parser = argparse.ArgumentParser(
             description='Helper tool for OS X proxy configuration and mitmproxy.',
-            epilog='Any additional arguments will be passed on unchanged to mitmproxy.'
+            epilog='Any additional arguments will be passed on unchanged to mitmproxy.')
-        )
+        parser.add_argument(
-        parser.add_argument('-t', '--toggle', action='store_true', help='just toggle the proxy configuration')
+            '-t',
+            '--toggle',
+            action='store_true',
+            help='just toggle the proxy configuration')
 #         parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy')
-        parser.add_argument('-p', '--port', type=int, help='override the default port of 8080', default=8080)
+        parser.add_argument(
+            '-p',
+            '--port',
+            type=int,
+            help='override the default port of 8080',
+            default=8080)
         args, extra_arguments = parser.parse_known_args()
 
         wrapper = cls(port=args.port, extra_arguments=extra_arguments)
@@ -139,4 +164,3 @@ class Wrapper(object):
 if __name__ == '__main__':
     Wrapper.ensure_superuser()
     Wrapper.main()
-

examples/modify_form.py

@@ -1,6 +1,7 @@
 
 def request(context, flow):
-    if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]:
+    if "application/x-www-form-urlencoded" in flow.request.headers[
+            "content-type"]:
         form = flow.request.get_form_urlencoded()
         form["mitmproxy"] = ["rocks"]
         flow.request.set_form_urlencoded(form)

examples/modify_response_body.py

@@ -6,10 +6,13 @@ from libmproxy.protocol.http import decoded
 def start(context, argv):
     if len(argv) != 3:
         raise ValueError('Usage: -s "modify-response-body.py old new"')
-    # You may want to use Python's argparse for more sophisticated argument parsing.
+    # You may want to use Python's argparse for more sophisticated argument
+    # parsing.
     context.old, context.new = argv[1], argv[2]
 
 
 def response(context, flow):
     with decoded(flow.response):  # automatically decode gzipped responses.
-        flow.response.content = flow.response.content.replace(context.old, context.new)
+        flow.response.content = flow.response.content.replace(
+            context.old,
+            context.new)

examples/read_dumpfile

@@ -4,7 +4,8 @@
 #
 
 from libmproxy import flow
-import json, sys
+import json
+import sys
 
 with open("logfile", "rb") as logfile:
     freader = flow.FlowReader(logfile)
@@ -14,5 +15,5 @@ with open("logfile", "rb") as logfile:
             print(f.request.host)
             json.dump(f.get_state(), sys.stdout, indent=4)
             print("")
-    except flow.FlowReadError, v:
+    except flow.FlowReadError as v:
-        print("Flow file corrupted. Stopped loading.")
+        print "Flow file corrupted. Stopped loading."

examples/redirect_requests.py

@@ -8,7 +8,8 @@ This example shows two ways to redirect flows to other destinations.
 
 def request(context, flow):
     # pretty_host(hostheader=True) takes the Host: header of the request into account,
-    # which is useful in transparent mode where we usually only have the IP otherwise.
+    # which is useful in transparent mode where we usually only have the IP
+    # otherwise.
 
     # Method 1: Answer with a locally generated response
     if flow.request.pretty_host(hostheader=True).endswith("example.com"):

examples/stub.py

@@ -1,12 +1,15 @@
 """
     This is a script stub, with definitions for all events.
 """
+
+
 def start(context, argv):
     """
         Called once on script startup, before any other events.
     """
     context.log("start")
 
+
 def clientconnect(context, conn_handler):
     """
         Called when a client initiates a connection to the proxy. Note that a
@@ -14,6 +17,7 @@ def clientconnect(context, conn_handler):
     """
     context.log("clientconnect")
 
+
 def serverconnect(context, conn_handler):
     """
         Called when the proxy initiates a connection to the target server. Note that a
@@ -21,6 +25,7 @@ def serverconnect(context, conn_handler):
     """
     context.log("serverconnect")
 
+
 def request(context, flow):
     """
         Called when a client request has been received.
@@ -36,12 +41,14 @@ def responseheaders(context, flow):
     """
     context.log("responseheaders")
 
+
 def response(context, flow):
     """
        Called when a server response has been received.
     """
     context.log("response")
 
+
 def error(context, flow):
     """
         Called when a flow error has occured, e.g. invalid server responses, or
@@ -50,12 +57,14 @@ def error(context, flow):
     """
     context.log("error")
 
+
 def clientdisconnect(context, conn_handler):
     """
         Called when a client disconnects from the proxy.
     """
     context.log("clientdisconnect")
 
+
 def done(context):
     """
         Called once on script shutdown, after any other events.

examples/upsidedownternet.py

@@ -2,6 +2,7 @@ import cStringIO
 from PIL import Image
 from libmproxy.protocol.http import decoded
 
+
 def response(context, flow):
     if flow.response.headers.get_first("content-type", "").startswith("image"):
         with decoded(flow.response):  # automatically decode gzipped responses.

libmproxy/cmdline.py

@@ -65,7 +65,7 @@ def parse_replace_hook(s):
     patt, regex, replacement = _parse_hook(s)
     try:
         re.compile(regex)
-    except re.error, e:
+    except re.error as e:
         raise ParseException("Malformed replacement regex: %s" % str(e.message))
     return patt, regex, replacement
 
@@ -127,7 +127,6 @@ def parse_server_spec_special(url):
     return ret
 
 
-
 def get_common_options(options):
     stickycookie, stickyauth = None, None
     if options.stickycookie_filt:
@@ -142,17 +141,17 @@ def get_common_options(options):
     for i in options.replace:
         try:
             p = parse_replace_hook(i)
-        except ParseException, e:
+        except ParseException as e:
             raise configargparse.ArgumentTypeError(e.message)
         reps.append(p)
     for i in options.replace_file:
         try:
             patt, rex, path = parse_replace_hook(i)
-        except ParseException, e:
+        except ParseException as e:
             raise configargparse.ArgumentTypeError(e.message)
         try:
             v = open(path, "rb").read()
-        except IOError, e:
+        except IOError as e:
             raise configargparse.ArgumentTypeError(
                 "Could not read replace file: %s" % path
             )
@@ -162,7 +161,7 @@ def get_common_options(options):
     for i in options.setheader:
         try:
             p = parse_setheader(i)
-        except ParseException, e:
+        except ParseException as e:
             raise configargparse.ArgumentTypeError(e.message)
         setheaders.append(p)
 
@@ -221,7 +220,7 @@ def common_options(parser):
     parser.add_argument(
         "--cadir",
         action="store", type=str, dest="cadir", default=config.CA_DIR,
-        help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR
+        help="Location of the default mitmproxy CA files. (%s)" % config.CA_DIR
     )
     parser.add_argument(
         "--host",
@@ -466,7 +465,7 @@ def common_options(parser):
         "--replay-ignore-payload-param",
         action="append", dest="replay_ignore_payload_params", type=str,
         help="""
-            Request's payload parameters (application/x-www-form-urlencoded) to
+            Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to
             be ignored while searching for a saved flow to replay.
             Can be passed multiple times.
         """
@@ -482,9 +481,10 @@ def common_options(parser):
     )
     group.add_argument(
         "--replay-ignore-host",
-        action="store_true", dest="replay_ignore_host", default=False,
+        action="store_true",
-        help="Ignore request's destination host while searching for a saved flow to replay"
+        dest="replay_ignore_host",
-    )
+        default=False,
+        help="Ignore request's destination host while searching for a saved flow to replay")
 
     group = parser.add_argument_group(
         "Replacements",
@@ -575,11 +575,16 @@ def mitmproxy():
     )
     common_options(parser)
     parser.add_argument(
-        "--palette", type=str, default="dark",
+        "--palette", type=str, default=palettes.DEFAULT,
         action="store", dest="palette",
         choices=sorted(palettes.palettes.keys()),
         help="Select color palette: " + ", ".join(palettes.palettes.keys())
     )
+    parser.add_argument(
+        "--palette-transparent",
+        action="store_true", dest="palette_transparent", default=False,
+        help="Set transparent background for palette."
+    )
     parser.add_argument(
         "-e", "--eventlog",
         action="store_true", dest="eventlog",
@@ -594,6 +599,11 @@ def mitmproxy():
         type=str, dest="intercept", default=None,
         help="Intercept filter expression."
     )
+    group.add_argument(
+        "-l", "--limit", action="store",
+        type=str, dest="limit", default=None,
+        help="Limit filter expression."
+    )
     return parser
 
 

libmproxy/console/common.py

@@ -6,15 +6,14 @@ import os
 
 from .. import utils
 from ..protocol.http import CONTENT_MISSING, decoded
+from . import signals
+import netlib.utils
 
 try:
     import pyperclip
 except:
     pyperclip = False
 
-VIEW_LIST = 0
-VIEW_FLOW = 1
-
 
 VIEW_FLOW_REQUEST = 0
 VIEW_FLOW_RESPONSE = 1
@@ -31,14 +30,22 @@ METHOD_OPTIONS = [
 ]
 
 
-def highlight_key(s, k):
+def is_keypress(k):
+    """
+        Is this input event a keypress?
+    """
+    if isinstance(k, basestring):
+        return True
+
+
+def highlight_key(str, key, textattr="text", keyattr="key"):
     l = []
-    parts = s.split(k, 1)
+    parts = str.split(key, 1)
     if parts[0]:
-        l.append(("text", parts[0]))
+        l.append((textattr, parts[0]))
-    l.append(("key", k))
+    l.append((keyattr, key))
     if parts[1]:
-        l.append(("text", parts[1]))
+        l.append((textattr, parts[1]))
     return l
 
 
@@ -60,20 +67,26 @@ def format_keyvals(lst, key="key", val="text", indent=0):
             if kv is None:
                 ret.append(urwid.Text(""))
             else:
-                cols = []
+                if isinstance(kv[1], urwid.Widget):
-                # This cumbersome construction process is here for a reason:
+                    v = kv[1]
-                # Urwid < 1.0 barfs if given a fixed size column of size zero.
+                elif kv[1] is None:
-                if indent:
+                    v = urwid.Text("")
-                    cols.append(("fixed", indent, urwid.Text("")))
+                else:
-                cols.extend([
+                    v = urwid.Text([(val, kv[1])])
+                ret.append(
+                    urwid.Columns(
+                        [
+                            ("fixed", indent, urwid.Text("")),
                             (
                                 "fixed",
                                 maxk,
                                 urwid.Text([(key, kv[0] or "")])
                             ),
-                    kv[1] if isinstance(kv[1], urwid.Widget) else urwid.Text([(val, kv[1])])
+                            v
-               ])
+                        ],
-                ret.append(urwid.Columns(cols, dividechars = 2))
+                        dividechars = 2
+                    )
+                )
     return ret
 
 
@@ -151,7 +164,7 @@ def raw_format_flow(f, focus, extended, padding):
             4: "code_400",
             5: "code_500",
         }
-        ccol = codes.get(f["resp_code"]/100, "code_other")
+        ccol = codes.get(f["resp_code"] / 100, "code_other")
         resp.append(fcol(SYMBOL_RETURN, ccol))
         if f["resp_is_replay"]:
             resp.append(fcol(SYMBOL_REPLAY, "replay"))
@@ -184,23 +197,39 @@ def raw_format_flow(f, focus, extended, padding):
 def save_data(path, data, master, state):
     if not path:
         return
-    state.last_saveload = path
-    path = os.path.expanduser(path)
     try:
         with file(path, "wb") as f:
             f.write(data)
-    except IOError, v:
+    except IOError as v:
-        master.statusbar.message(v.strerror)
+        signals.status_message.send(message=v.strerror)
+
+
+def ask_save_overwite(path, data, master, state):
+    if not path:
+        return
+    path = os.path.expanduser(path)
+    if os.path.exists(path):
+        def save_overwite(k):
+            if k == "y":
+                save_data(path, data, master, state)
+
+        signals.status_prompt_onekey.send(
+            prompt = "'" + path + "' already exists. Overwite?",
+            keys = (
+                ("yes", "y"),
+                ("no", "n"),
+            ),
+            callback = save_overwite
+        )
+    else:
+        save_data(path, data, master, state)
 
 
 def ask_save_path(prompt, data, master, state):
-    master.path_prompt(
+    signals.status_prompt_path.send(
-        prompt,
+        prompt = prompt,
-        state.last_saveload,
+        callback = ask_save_overwite,
-        save_data,
+        args = (data, master, state)
-        data,
-        master,
-        state
     )
 
 
@@ -210,6 +239,8 @@ def copy_flow_format_data(part, scope, flow):
     else:
         data = ""
         if scope in ("q", "a"):
+            if flow.request.content is None or flow.request.content == CONTENT_MISSING:
+                return None, "Request content is missing"
             with decoded(flow.request):
                 if part == "h":
                     data += flow.request.assemble()
@@ -221,6 +252,8 @@ def copy_flow_format_data(part, scope, flow):
             # Add padding between request and response
             data += "\r\n" * 2
         if scope in ("s", "a") and flow.response:
+            if flow.response.content is None or flow.response.content == CONTENT_MISSING:
+                return None, "Response content is missing"
             with decoded(flow.response):
                 if part == "h":
                     data += flow.response.assemble()
@@ -228,40 +261,43 @@ def copy_flow_format_data(part, scope, flow):
                     data += flow.response.content
                 else:
                     raise ValueError("Unknown part: {}".format(part))
-    return data
+    return data, False
 
 
 def copy_flow(part, scope, flow, master, state):
     """
-    part: _c_ontent, _a_ll, _u_rl
+    part: _c_ontent, _h_eaders+content, _u_rl
     scope: _a_ll, re_q_uest, re_s_ponse
     """
-    data = copy_flow_format_data(part, scope, flow)
+    data, err = copy_flow_format_data(part, scope, flow)
+
+    if err:
+        signals.status_message.send(message=err)
+        return
 
     if not data:
         if scope == "q":
-            master.statusbar.message("No request content to copy.")
+            signals.status_message.send(message="No request content to copy.")
         elif scope == "s":
-            master.statusbar.message("No response content to copy.")
+            signals.status_message.send(message="No response content to copy.")
         else:
-            master.statusbar.message("No contents to copy.")
+            signals.status_message.send(message="No contents to copy.")
         return
 
     try:
         master.add_event(str(len(data)))
         pyperclip.copy(data)
-    except RuntimeError:
+    except (RuntimeError, UnicodeDecodeError):
         def save(k):
             if k == "y":
-                ask_save_path("Save data: ", data, master, state)
+                ask_save_path("Save data", data, master, state)
-
+        signals.status_prompt_onekey.send(
-        master.prompt_onekey(
+            prompt = "Cannot copy binary data to clipboard. Save as file?",
-            "Cannot copy binary data to clipboard. Save as file?",
+            keys = (
-            (
                 ("yes", "y"),
                 ("no", "n"),
             ),
-            save
+            callback = save
         )
 
 
@@ -273,14 +309,11 @@ def ask_copy_part(scope, flow, master, state):
     if scope != "s":
         choices.append(("url", "u"))
 
-    master.prompt_onekey(
+    signals.status_prompt_onekey.send(
-        "Copy",
+        prompt = "Copy",
-        choices,
+        keys = choices,
-        copy_flow,
+        callback = copy_flow,
-        scope,
+        args = (scope, flow, master, state)
-        flow,
-        master,
-        state
     )
 
 
@@ -297,16 +330,14 @@ def ask_save_body(part, master, state, flow):
         # We first need to determine whether we want to save the request or the
         # response content.
         if request_has_content and response_has_content:
-            master.prompt_onekey(
+            signals.status_prompt_onekey.send(
-                "Save",
+                prompt = "Save",
-                (
+                keys = (
                     ("request", "q"),
                     ("response", "s"),
                 ),
-                ask_save_body,
+                callback = ask_save_body,
-                master,
+                args = (master, state, flow)
-                state,
-                flow
             )
         elif response_has_content:
             ask_save_body("s", master, state, flow)
@@ -315,27 +346,23 @@ def ask_save_body(part, master, state, flow):
 
     elif part == "q" and request_has_content:
         ask_save_path(
-            "Save request content: ",
+            "Save request content",
             flow.request.get_decoded_content(),
             master,
             state
         )
     elif part == "s" and response_has_content:
         ask_save_path(
-            "Save response content: ",
+            "Save response content",
             flow.response.get_decoded_content(),
             master,
             state
         )
     else:
-        master.statusbar.message("No content to save.")
+        signals.status_message.send(message="No content to save.")
 
 
-class FlowCache:
+flowcache = utils.LRUCache(800)
-    @utils.LRUCache(200)
-    def format_flow(self, *args):
-        return raw_format_flow(*args)
-flowcache = FlowCache()
 
 
 def format_flow(f, focus, extended=False, hostheader=False, padding=2):
@@ -353,7 +380,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
     )
     if f.response:
         if f.response.content:
-            contentdesc = utils.pretty_size(len(f.response.content))
+            contentdesc = netlib.utils.pretty_size(len(f.response.content))
         elif f.response.content == CONTENT_MISSING:
             contentdesc = "[content missing]"
         else:
@@ -374,6 +401,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
             d["resp_ctype"] = t[0].split(";")[0]
         else:
             d["resp_ctype"] = ""
-    return flowcache.format_flow(
+    return flowcache.get(
+        raw_format_flow,
         tuple(sorted(d.items())), focus, extended, padding
     )

libmproxy/console/contentview.py

@@ -6,15 +6,15 @@ import lxml.html
 import lxml.etree
 from PIL import Image
 from PIL.ExifTags import TAGS
-import re
 import subprocess
 import traceback
 import urwid
 
 import netlib.utils
+from netlib import odict
 
 from . import common
-from .. import utils, encoding, flow
+from .. import utils, encoding
 from ..contrib import jsbeautifier, html2text
 from ..contrib.wbxml.ASCommandResponse import ASCommandResponse
 
@@ -36,7 +36,7 @@ else:
     cssutils.ser.prefs.indentClosingBrace = False
     cssutils.ser.prefs.validOnly = False
 
-VIEW_CUTOFF = 1024*50
+VIEW_CUTOFF = 1024 * 50
 
 
 def _view_text(content, total, limit):
@@ -59,7 +59,7 @@ def trailer(clen, txt, limit):
         txt.append(
             urwid.Text(
                 [
-                    ("highlight", "... %s of data not shown. Press "%utils.pretty_size(rem)),
+                    ("highlight", "... %s of data not shown. Press " % netlib.utils.pretty_size(rem)),
                     ("key", "f"),
                     ("highlight", " to load all data.")
                 ]
@@ -76,7 +76,7 @@ class ViewAuto:
         ctype = hdrs.get_first("content-type")
         if ctype:
             ct = utils.parse_content_type(ctype) if ctype else None
-            ct = "%s/%s"%(ct[0], ct[1])
+            ct = "%s/%s" % (ct[0], ct[1])
             if ct in content_types_map:
                 return content_types_map[ct][0](hdrs, content, limit)
             elif utils.isXML(content):
@@ -227,7 +227,7 @@ class ViewURLEncoded:
         lines = utils.urldecode(content)
         if lines:
             body = common.format_keyvals(
-                [(k+":", v) for (k, v) in lines],
+                [(k + ":", v) for (k, v) in lines],
                 key = "header",
                 val = "text"
             )
@@ -240,33 +240,13 @@ class ViewMultipart:
     content_types = ["multipart/form-data"]
 
     def __call__(self, hdrs, content, limit):
-        v = hdrs.get_first("content-type")
+        v = utils.multipartdecode(hdrs, content)
         if v:
-            v = utils.parse_content_type(v)
-            if not v:
-                return
-            boundary = v[2].get("boundary")
-            if not boundary:
-                return
-
-            rx = re.compile(r'\bname="([^"]+)"')
-            keys = []
-            vals = []
-
-            for i in content.split("--" + boundary):
-                parts = i.splitlines()
-                if len(parts) > 1 and parts[0][0:2] != "--":
-                    match = rx.search(parts[1])
-                    if match:
-                        keys.append(match.group(1) + ":")
-                        vals.append(netlib.utils.cleanBin(
-                            "\n".join(parts[3+parts[2:].index(""):])
-                        ))
             r = [
                 urwid.Text(("highlight", "Form data:\n")),
             ]
             r.extend(common.format_keyvals(
-                zip(keys, vals),
+                v,
                 key = "header",
                 val = "text"
             ))
@@ -324,7 +304,6 @@ if pyamf:
             if not envelope:
                 return None
 
-
             txt = []
             for target, message in iter(envelope):
                 if isinstance(message, pyamf.remoting.Request):
@@ -335,13 +314,13 @@ if pyamf:
                 else:
                     txt.append(urwid.Text([
                         ("header", "Response: "),
-                        ("text", "%s, code %s"%(target, message.status)),
+                        ("text", "%s, code %s" % (target, message.status)),
                     ]))
 
                 s = json.dumps(self.unpack(message), indent=4)
                 txt.extend(_view_text(s[:limit], len(s), limit))
 
-            return "AMF v%s"%envelope.amfVersion, txt
+            return "AMF v%s" % envelope.amfVersion, txt
 
 
 class ViewJavaScript:
@@ -395,7 +374,7 @@ class ViewImage:
             return None
         parts = [
             ("Format", str(img.format_description)),
-            ("Size", "%s x %s px"%img.size),
+            ("Size", "%s x %s px" % img.size),
             ("Mode", str(img.mode)),
         ]
         for i in sorted(img.info.keys()):
@@ -421,7 +400,7 @@ class ViewImage:
             key = "header",
             val = "text"
         )
-        return "%s image"%img.format, fmt
+        return "%s image" % img.format, fmt
 
 
 class ViewProtobuf:
@@ -539,14 +518,14 @@ def get_content_view(viewmode, hdrItems, content, limit, logfunc, is_request):
             return "No content", ""
     msg = []
 
-    hdrs = flow.ODictCaseless([list(i) for i in hdrItems])
+    hdrs = odict.ODictCaseless([list(i) for i in hdrItems])
 
     enc = hdrs.get_first("content-encoding")
     if enc and enc != "identity":
         decoded = encoding.decode(enc, content)
         if decoded:
             content = decoded
-            msg.append("[decoded %s]"%enc)
+            msg.append("[decoded %s]" % enc)
     try:
         ret = viewmode(hdrs, content, limit)
     # Third-party viewers can fail in unexpected ways...

libmproxy/console/flowdetailview.py

@@ -1,44 +1,24 @@
 from __future__ import absolute_import
 import urwid
-from . import common
+from . import common, searchable
 from .. import utils
 
-footer = [
-    ('heading_key', "q"), ":back ",
-]
 
-class FlowDetailsView(urwid.ListBox):
+def maybe_timestamp(base, attr):
-    def __init__(self, master, flow, state):
+    if base and getattr(base, attr):
-        self.master, self.flow, self.state = master, flow, state
+        return utils.format_timestamp_with_milli(getattr(base, attr))
-        urwid.ListBox.__init__(
+    else:
-            self,
+        return "active"
-            self.flowtext()
+    pass
-        )
 
-    def keypress(self, size, key):
-        key = common.shortcuts(key)
-        if key == "q":
-            self.master.statusbar = self.state[0]
-            self.master.body = self.state[1]
-            self.master.header = self.state[2]
-            self.master.loop.widget = self.master.make_view()
-            return None
-        elif key == "?":
-            key = None
-        return urwid.ListBox.keypress(self, size, key)
 
-    def flowtext(self):
+def flowdetails(state, flow):
     text = []
 
-        title = urwid.Text("Flow details")
+    cc = flow.client_conn
-        title = urwid.Padding(title, align="left", width=("relative", 100))
+    sc = flow.server_conn
-        title = urwid.AttrWrap(title, "heading")
+    req = flow.request
-        text.append(title)
+    resp = flow.response
-
-        cc = self.flow.client_conn
-        sc = self.flow.server_conn
-        req = self.flow.request
-        resp = self.flow.response
 
     if sc:
         text.append(urwid.Text([("head", "Server Connection:")]))
@@ -46,13 +26,15 @@ class FlowDetailsView(urwid.ListBox):
             ["Address", "%s:%s" % sc.address()],
         ]
 
-            text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+        text.extend(
+            common.format_keyvals(parts, key="key", val="text", indent=4)
+        )
 
         c = sc.cert
         if c:
             text.append(urwid.Text([("head", "Server Certificate:")]))
             parts = [
-                    ["Type", "%s, %s bits"%c.keyinfo],
+                ["Type", "%s, %s bits" % c.keyinfo],
                 ["SHA1 digest", c.digest("sha1")],
                 ["Valid to", str(c.notafter)],
                 ["Valid from", str(c.notbefore)],
@@ -60,14 +42,24 @@ class FlowDetailsView(urwid.ListBox):
                 [
                     "Subject",
                     urwid.BoxAdapter(
-                            urwid.ListBox(common.format_keyvals(c.subject, key="highlight", val="text")),
+                        urwid.ListBox(
+                            common.format_keyvals(
+                                c.subject,
+                                key="highlight",
+                                val="text"
+                            )
+                        ),
                         len(c.subject)
                     )
                 ],
                 [
                     "Issuer",
                     urwid.BoxAdapter(
-                            urwid.ListBox(common.format_keyvals(c.issuer, key="highlight", val="text")),
+                        urwid.ListBox(
+                            common.format_keyvals(
+                                c.issuer, key="highlight", val="text"
+                            )
+                        ),
                         len(c.issuer)
                     )
                 ]
@@ -80,7 +72,9 @@ class FlowDetailsView(urwid.ListBox):
                         ", ".join(c.altnames)
                     ]
                 )
-                text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+            text.extend(
+                common.format_keyvals(parts, key="key", val="text", indent=4)
+            )
 
     if cc:
         text.append(urwid.Text([("head", "Client Connection:")]))
@@ -90,24 +84,71 @@ class FlowDetailsView(urwid.ListBox):
             # ["Requests", "%s"%cc.requestcount],
         ]
 
-            text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+        text.extend(
+            common.format_keyvals(parts, key="key", val="text", indent=4)
+        )
 
     parts = []
 
-        parts.append(["Client conn. established", utils.format_timestamp_with_milli(cc.timestamp_start) if (cc and cc.timestamp_start) else "active"])
+    parts.append(
-        parts.append(["Server conn. initiated", utils.format_timestamp_with_milli(sc.timestamp_start) if sc else "active" ])
+        [
-        parts.append(["Server conn. TCP handshake", utils.format_timestamp_with_milli(sc.timestamp_tcp_setup) if (sc and sc.timestamp_tcp_setup) else "active"])
+            "Client conn. established",
+            maybe_timestamp(cc, "timestamp_start")
+        ]
+    )
+    parts.append(
+        [
+            "Server conn. initiated",
+            maybe_timestamp(sc, "timestamp_start")
+        ]
+    )
+    parts.append(
+        [
+            "Server conn. TCP handshake",
+            maybe_timestamp(sc, "timestamp_tcp_setup")
+        ]
+    )
     if sc.ssl_established:
-            parts.append(["Server conn. SSL handshake", utils.format_timestamp_with_milli(sc.timestamp_ssl_setup) if sc.timestamp_ssl_setup else "active"])
+        parts.append(
-            parts.append(["Client conn. SSL handshake", utils.format_timestamp_with_milli(cc.timestamp_ssl_setup) if (cc and cc.timestamp_ssl_setup) else "active"])
+            [
-        parts.append(["First request byte", utils.format_timestamp_with_milli(req.timestamp_start)])
+                "Server conn. SSL handshake",
-        parts.append(["Request complete", utils.format_timestamp_with_milli(req.timestamp_end) if req.timestamp_end else "active"])
+                maybe_timestamp(sc, "timestamp_ssl_setup")
-        parts.append(["First response byte", utils.format_timestamp_with_milli(resp.timestamp_start) if resp else "active"])
+            ]
-        parts.append(["Response complete", utils.format_timestamp_with_milli(resp.timestamp_end) if (resp and resp.timestamp_end) else "active"])
+        )
+        parts.append(
+            [
+                "Client conn. SSL handshake",
+                maybe_timestamp(cc, "timestamp_ssl_setup")
+            ]
+        )
+    parts.append(
+        [
+            "First request byte",
+            maybe_timestamp(req, "timestamp_start")
+        ]
+    )
+    parts.append(
+        [
+            "Request complete",
+            maybe_timestamp(req, "timestamp_end")
+        ]
+    )
+    parts.append(
+        [
+            "First response byte",
+            maybe_timestamp(resp, "timestamp_start")
+        ]
+    )
+    parts.append(
+        [
+            "Response complete",
+            maybe_timestamp(resp, "timestamp_end")
+        ]
+    )
 
     # sort operations by timestamp
     parts = sorted(parts, key=lambda p: p[1])
 
     text.append(urwid.Text([("head", "Timing:")]))
     text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
-        return text
+    return searchable.Searchable(state, text)

libmproxy/console/flowlist.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 import urwid
 from netlib import http
-from . import common
+from . import common, signals
 
 
 def _mkhelp():
@@ -15,10 +15,10 @@ def _mkhelp():
         ("D", "duplicate flow"),
         ("e", "toggle eventlog"),
         ("F", "toggle follow flow list"),
-        ("g", "copy flow to clipboard"),
         ("l", "set limit filter pattern"),
         ("L", "load saved flows"),
         ("n", "create a new request"),
+        ("P", "copy flow to clipboard"),
         ("r", "replay request"),
         ("V", "revert changes to request"),
         ("w", "save flows "),
@@ -47,6 +47,10 @@ class EventListBox(urwid.ListBox):
         if key == "C":
             self.master.clear_events()
             key = None
+        elif key == "G":
+            self.set_focus(0)
+        elif key == "g":
+            self.set_focus(len(self.master.eventlist) - 1)
         return urwid.ListBox.keypress(self, size, key)
 
 
@@ -72,7 +76,8 @@ class BodyPile(urwid.Pile):
 
     def keypress(self, size, key):
         if key == "tab":
-            self.focus_position = (self.focus_position + 1)%len(self.widget_list)
+            self.focus_position = (
+                self.focus_position + 1) % len(self.widget_list)
             if self.focus_position == 1:
                 self.widget_list[1].header = self.active_header
             else:
@@ -111,17 +116,15 @@ class ConnectionItem(urwid.WidgetWrap):
 
     def save_flows_prompt(self, k):
         if k == "a":
-            self.master.path_prompt(
+            signals.status_prompt_path.send(
-                "Save all flows to: ",
+                prompt = "Save all flows to",
-                self.state.last_saveload,
+                callback = self.master.save_flows
-                self.master.save_flows
             )
         else:
-            self.master.path_prompt(
+            signals.status_prompt_path.send(
-                "Save this flow to: ",
+                prompt = "Save this flow to",
-                self.state.last_saveload,
+                callback = self.master.save_one_flow,
-                self.master.save_one_flow,
+                args = (self.flow,)
-                self.flow
             )
 
     def stop_server_playback_prompt(self, a):
@@ -150,64 +153,65 @@ class ConnectionItem(urwid.WidgetWrap):
                 self.master.options.replay_ignore_host
             )
         else:
-            self.master.path_prompt(
+            signals.status_prompt_path.send(
-                "Server replay path: ",
+                prompt = "Server replay path",
-                self.state.last_saveload,
+                callback = self.master.server_playback_path
-                self.master.server_playback_path
             )
 
-    def keypress(self, (maxcol,), key):
+    def keypress(self, xxx_todo_changeme, key):
+        (maxcol,) = xxx_todo_changeme
         key = common.shortcuts(key)
         if key == "a":
             self.flow.accept_intercept(self.master)
-            self.master.sync_list_view()
+            signals.flowlist_change.send(self)
         elif key == "d":
             self.flow.kill(self.master)
             self.state.delete_flow(self.flow)
-            self.master.sync_list_view()
+            signals.flowlist_change.send(self)
         elif key == "D":
             f = self.master.duplicate_flow(self.flow)
             self.master.view_flow(f)
         elif key == "r":
             r = self.master.replay_request(self.flow)
             if r:
-                self.master.statusbar.message(r)
+                signals.status_message.send(message=r)
-            self.master.sync_list_view()
+            signals.flowlist_change.send(self)
         elif key == "S":
             if not self.master.server_playback:
-                self.master.prompt_onekey(
+                signals.status_prompt_onekey.send(
-                    "Server Replay",
+                    prompt = "Server Replay",
-                    (
+                    keys = (
                         ("all flows", "a"),
                         ("this flow", "t"),
                         ("file", "f"),
                     ),
-                    self.server_replay_prompt,
+                    callback = self.server_replay_prompt,
                 )
             else:
-                self.master.prompt_onekey(
+                signals.status_prompt_onekey.send(
-                    "Stop current server replay?",
+                    prompt = "Stop current server replay?",
-                    (
+                    keys = (
                         ("yes", "y"),
                         ("no", "n"),
                     ),
-                    self.stop_server_playback_prompt,
+                    callback = self.stop_server_playback_prompt,
                 )
         elif key == "V":
             if not self.flow.modified():
-                self.master.statusbar.message("Flow not modified.")
+                signals.status_message.send(message="Flow not modified.")
                 return
             self.state.revert(self.flow)
-            self.master.sync_list_view()
+            signals.flowlist_change.send(self)
-            self.master.statusbar.message("Reverted.")
+            signals.status_message.send(message="Reverted.")
         elif key == "w":
-            self.master.prompt_onekey(
+            signals.status_prompt_onekey.send(
-                "Save",
+                self,
-                (
+                prompt = "Save",
+                keys = (
                     ("all flows", "a"),
                     ("this flow", "t"),
                 ),
-                self.save_flows_prompt,
+                callback = self.save_flows_prompt,
             )
         elif key == "X":
             self.flow.kill(self.master)
@@ -215,13 +219,12 @@ class ConnectionItem(urwid.WidgetWrap):
             if self.flow.request:
                 self.master.view_flow(self.flow)
         elif key == "|":
-            self.master.path_prompt(
+            signals.status_prompt_path.send(
-                "Send flow to script: ",
+                prompt = "Send flow to script",
-                self.state.last_script,
+                callback = self.master.run_script_once,
-                self.master.run_script_once,
+                args = (self.flow,)
-                self.flow
             )
-        elif key == "g":
+        elif key == "P":
             common.ask_copy_part("a", self.flow, self.master, self.state)
         elif key == "b":
             common.ask_save_body(None, self.master, self.state, self.flow)
@@ -232,8 +235,10 @@ class ConnectionItem(urwid.WidgetWrap):
 class FlowListWalker(urwid.ListWalker):
     def __init__(self, master, state):
         self.master, self.state = master, state
-        if self.state.flow_count():
+        signals.flowlist_change.connect(self.sig_flowlist_change)
-            self.set_focus(0)
+
+    def sig_flowlist_change(self, sender):
+        self._modified()
 
     def get_focus(self):
         f, i = self.state.get_focus()
@@ -258,7 +263,10 @@ class FlowListWalker(urwid.ListWalker):
 class FlowListBox(urwid.ListBox):
     def __init__(self, master):
         self.master = master
-        urwid.ListBox.__init__(self, master.flow_list_walker)
+        urwid.ListBox.__init__(
+            self,
+            FlowListWalker(master, master.state)
+        )
 
     def get_method_raw(self, k):
         if k:
@@ -266,7 +274,12 @@ class FlowListBox(urwid.ListBox):
 
     def get_method(self, k):
         if k == "e":
-            self.master.prompt("Method:", "", self.get_method_raw)
+            signals.status_prompt.send(
+                self,
+                prompt = "Method",
+                text = "",
+                callback = self.get_method_raw
+            )
         else:
             method = ""
             for i in common.METHOD_OPTIONS:
@@ -275,17 +288,17 @@ class FlowListBox(urwid.ListBox):
             self.get_url(method)
 
     def get_url(self, method):
-        self.master.prompt(
+        signals.status_prompt.send(
-            "URL:",
+            prompt = "URL",
-            "http://www.example.com/",
+            text = "http://www.example.com/",
-            self.new_request,
+            callback = self.new_request,
-            method
+            args = (method,)
         )
 
     def new_request(self, url, method):
         parts = http.parse_url(str(url))
         if not parts:
-            self.master.statusbar.message("Invalid Url")
+            signals.status_message.send(message="Invalid Url")
             return
         scheme, host, port, path = parts
         f = self.master.create_request(method, scheme, host, port, path)
@@ -295,28 +308,34 @@ class FlowListBox(urwid.ListBox):
         key = common.shortcuts(key)
         if key == "A":
             self.master.accept_all()
-            self.master.sync_list_view()
+            signals.flowlist_change.send(self)
         elif key == "C":
             self.master.clear_flows()
         elif key == "e":
             self.master.toggle_eventlog()
+        elif key == "G":
+            self.master.state.set_focus(0)
+            signals.flowlist_change.send(self)
+        elif key == "g":
+            self.master.state.set_focus(self.master.state.flow_count())
+            signals.flowlist_change.send(self)
         elif key == "l":
-            self.master.prompt(
+            signals.status_prompt.send(
-                "Limit: ",
+                prompt = "Limit",
-                self.master.state.limit_txt,
+                text = self.master.state.limit_txt,
-                self.master.set_limit
+                callback = self.master.set_limit
             )
         elif key == "L":
-            self.master.path_prompt(
+            signals.status_prompt_path.send(
-                "Load flows: ",
+                self,
-                self.master.state.last_saveload,
+                prompt = "Load flows",
-                self.master.load_flows_callback
+                callback = self.master.load_flows_callback
             )
         elif key == "n":
-            self.master.prompt_onekey(
+            signals.status_prompt_onekey.send(
-                "Method",
+                prompt = "Method",
-                common.METHOD_OPTIONS,
+                keys = common.METHOD_OPTIONS,
-                self.get_method
+                callback = self.get_method
             )
         elif key == "F":
             self.master.toggle_follow_flows()
@@ -324,10 +343,10 @@ class FlowListBox(urwid.ListBox):
             if self.master.stream:
                 self.master.stop_stream()
             else:
-                self.master.path_prompt(
+                signals.status_prompt_path.send(
-                    "Stream flows to: ",
+                    self,
-                    self.master.state.last_saveload,
+                    prompt = "Stream flows to",
-                    self.master.start_stream_to_path
+                    callback = self.master.start_stream_to_path
                 )
         else:
             return urwid.ListBox.keypress(self, size, key)

libmproxy/console/grideditor.py

@@ -5,31 +5,99 @@ import re
 import os
 import urwid
 
-from . import common
+from . import common, signals
 from .. import utils, filt, script
-from netlib import http_uastrings
+from netlib import http_uastrings, http_cookies, odict
 
 
-footer = [
+FOOTER = [
     ('heading_key', "enter"), ":edit ",
     ('heading_key', "q"), ":back ",
 ]
-footer_editing = [
+FOOTER_EDITING = [
     ('heading_key', "esc"), ":stop editing ",
 ]
 
 
-class SText(urwid.WidgetWrap):
+class TextColumn:
-    def __init__(self, txt, focused, error):
+    subeditor = None
+
+    def __init__(self, heading):
+        self.heading = heading
+
+    def text(self, obj):
+        return SEscaped(obj or "")
+
+    def blank(self):
+        return ""
+
+    def keypress(self, key, editor):
+        if key == "r":
+            if editor.walker.get_current_value() is not None:
+                signals.status_prompt_path.send(
+                    self,
+                    prompt = "Read file",
+                    callback = editor.read_file
+                )
+        elif key == "R":
+            if editor.walker.get_current_value() is not None:
+                signals.status_prompt_path.send(
+                    editor,
+                    prompt = "Read unescaped file",
+                    callback = editor.read_file,
+                    args = (True,)
+                )
+        elif key == "e":
+            o = editor.walker.get_current_value()
+            if o is not None:
+                n = editor.master.spawn_editor(o.encode("string-escape"))
+                n = utils.clean_hanging_newline(n)
+                editor.walker.set_current_value(n, False)
+                editor.walker._modified()
+        elif key in ["enter"]:
+            editor.walker.start_edit()
+        else:
+            return key
+
+
+class SubgridColumn:
+    def __init__(self, heading, subeditor):
+        self.heading = heading
+        self.subeditor = subeditor
+
+    def text(self, obj):
+        p = http_cookies._format_pairs(obj, sep="\n")
+        return urwid.Text(p)
+
+    def blank(self):
+        return []
+
+    def keypress(self, key, editor):
+        if key in "rRe":
+            signals.status_message.send(
+                self,
+                message = "Press enter to edit this field.",
+                expire = 1000
+            )
+            return
+        elif key in ["enter"]:
+            editor.master.view_grideditor(
+                self.subeditor(
+                    editor.master,
+                    editor.walker.get_current_value(),
+                    editor.set_subeditor_value,
+                    editor.walker.focus,
+                    editor.walker.focus_col
+                )
+            )
+        else:
+            return key
+
+
+class SEscaped(urwid.WidgetWrap):
+    def __init__(self, txt):
         txt = txt.encode("string-escape")
         w = urwid.Text(txt, wrap="any")
-        if focused:
-            if error:
-                w = urwid.AttrWrap(w, "focusfield_error")
-            else:
-                w = urwid.AttrWrap(w, "focusfield")
-        elif error:
-            w = urwid.AttrWrap(w, "field_error")
         urwid.WidgetWrap.__init__(self, w)
 
     def get_text(self):
@@ -50,7 +118,7 @@ class SEdit(urwid.WidgetWrap):
         urwid.WidgetWrap.__init__(self, w)
 
     def get_text(self):
-        return self._w.get_text()[0]
+        return self._w.get_text()[0].strip()
 
     def selectable(self):
         return True
@@ -67,9 +135,15 @@ class GridRow(urwid.WidgetWrap):
                 self.editing = SEdit(v)
                 self.fields.append(self.editing)
             else:
-                self.fields.append(
+                w = self.editor.columns[i].text(v)
-                    SText(v, True if focused == i else False, i in errors)
+                if focused == i:
-                )
+                    if i in errors:
+                        w = urwid.AttrWrap(w, "focusfield_error")
+                    else:
+                        w = urwid.AttrWrap(w, "focusfield")
+                elif i in errors:
+                    w = urwid.AttrWrap(w, "field_error")
+                self.fields.append(w)
 
         fspecs = self.fields[:]
         if len(self.fields) > 1:
@@ -101,6 +175,7 @@ class GridWalker(urwid.ListWalker):
         and errors is a set with an entry of each offset in rows that is an
         error.
     """
+
     def __init__(self, lst, editor):
         self.lst = [(i, set([])) for i in lst]
         self.editor = editor
@@ -125,31 +200,43 @@ class GridWalker(urwid.ListWalker):
             try:
                 val = val.decode("string-escape")
             except ValueError:
-                self.editor.master.statusbar.message(
+                signals.status_message.send(
-                    "Invalid Python-style string encoding.", 1000
+                    self,
+                    message = "Invalid Python-style string encoding.",
+                    expire = 1000
                 )
                 return
         errors = self.lst[self.focus][1]
         emsg = self.editor.is_error(self.focus_col, val)
         if emsg:
-            self.editor.master.statusbar.message(emsg, 1000)
+            signals.status_message.send(message = emsg, expire = 1)
             errors.add(self.focus_col)
         else:
             errors.discard(self.focus_col)
+        self.set_value(val, self.focus, self.focus_col, errors)
 
-        row = list(self.lst[self.focus][0])
+    def set_value(self, val, focus, focus_col, errors=None):
-        row[self.focus_col] = val
+        if not errors:
-        self.lst[self.focus] = [tuple(row), errors]
+            errors = set([])
+        row = list(self.lst[focus][0])
+        row[focus_col] = val
+        self.lst[focus] = [tuple(row), errors]
+        self._modified()
 
     def delete_focus(self):
         if self.lst:
             del self.lst[self.focus]
-            self.focus = min(len(self.lst)-1, self.focus)
+            self.focus = min(len(self.lst) - 1, self.focus)
             self._modified()
 
     def _insert(self, pos):
         self.focus = pos
-        self.lst.insert(self.focus, [[""]*self.editor.columns, set([])])
+        self.lst.insert(
+            self.focus,
+            [
+                [c.blank() for c in self.editor.columns], set([])
+            ]
+        )
         self.focus_col = 0
         self.start_edit()
 
@@ -160,16 +247,17 @@ class GridWalker(urwid.ListWalker):
         return self._insert(min(self.focus + 1, len(self.lst)))
 
     def start_edit(self):
-        if self.lst:
+        col = self.editor.columns[self.focus_col]
+        if self.lst and not col.subeditor:
             self.editing = GridRow(
                 self.focus_col, True, self.editor, self.lst[self.focus]
             )
-            self.editor.master.statusbar.update(footer_editing)
+            self.editor.master.loop.widget.footer.update(FOOTER_EDITING)
             self._modified()
 
     def stop_edit(self):
         if self.editing:
-            self.editor.master.statusbar.update(footer)
+            self.editor.master.loop.widget.footer.update(FOOTER)
             self.set_current_value(self.editing.get_edit_value(), False)
             self.editing = False
             self._modified()
@@ -179,14 +267,14 @@ class GridWalker(urwid.ListWalker):
         self._modified()
 
     def right(self):
-        self.focus_col = min(self.focus_col + 1, self.editor.columns-1)
+        self.focus_col = min(self.focus_col + 1, len(self.editor.columns) - 1)
         self._modified()
 
     def tab_next(self):
         self.stop_edit()
-        if self.focus_col < self.editor.columns-1:
+        if self.focus_col < len(self.editor.columns) - 1:
             self.focus_col += 1
-        elif self.focus != len(self.lst)-1:
+        elif self.focus != len(self.lst) - 1:
             self.focus_col = 0
             self.focus += 1
         self._modified()
@@ -207,16 +295,17 @@ class GridWalker(urwid.ListWalker):
     def set_focus(self, focus):
         self.stop_edit()
         self.focus = focus
+        self._modified()
 
     def get_next(self, pos):
-        if pos+1 >= len(self.lst):
+        if pos + 1 >= len(self.lst):
             return None, None
-        return GridRow(None, False, self.editor, self.lst[pos+1]), pos+1
+        return GridRow(None, False, self.editor, self.lst[pos + 1]), pos + 1
 
     def get_prev(self, pos):
-        if pos-1 < 0:
+        if pos - 1 < 0:
             return None, None
-        return GridRow(None, False, self.editor, self.lst[pos-1]), pos-1
+        return GridRow(None, False, self.editor, self.lst[pos - 1]), pos - 1
 
 
 class GridListBox(urwid.ListBox):
@@ -231,17 +320,16 @@ FIRST_WIDTH_MIN = 20
 class GridEditor(urwid.WidgetWrap):
     title = None
     columns = None
-    headings = None
 
     def __init__(self, master, value, callback, *cb_args, **cb_kwargs):
-        value = copy.deepcopy(value)
+        value = self.data_in(copy.deepcopy(value))
         self.master, self.value, self.callback = master, value, callback
         self.cb_args, self.cb_kwargs = cb_args, cb_kwargs
 
         first_width = 20
         if value:
             for r in value:
-                assert len(r) == self.columns
+                assert len(r) == len(self.columns)
                 first_width = max(len(r), first_width)
         self.first_width = min(first_width, FIRST_WIDTH_MAX)
 
@@ -250,9 +338,9 @@ class GridEditor(urwid.WidgetWrap):
         title = urwid.AttrWrap(title, "heading")
 
         headings = []
-        for i, h in enumerate(self.headings):
+        for i, col in enumerate(self.columns):
-            c = urwid.Text(h)
+            c = urwid.Text(col.heading)
-            if i == 0 and len(self.headings) > 1:
+            if i == 0 and len(self.columns) > 1:
                 headings.append(("fixed", first_width + 2, c))
             else:
                 headings.append(c)
@@ -268,7 +356,7 @@ class GridEditor(urwid.WidgetWrap):
             self.lb,
             header = urwid.Pile([title, h])
         )
-        self.master.statusbar.update("")
+        self.master.loop.widget.footer.update("")
         self.show_empty_msg()
 
     def show_empty_msg(self):
@@ -300,9 +388,12 @@ class GridEditor(urwid.WidgetWrap):
                 d = file(p, "rb").read()
                 self.walker.set_current_value(d, unescaped)
                 self.walker._modified()
-            except IOError, v:
+            except IOError as v:
                 return str(v)
 
+    def set_subeditor_value(self, val, focus, focus_col):
+        self.walker.set_value(val, focus, focus_col)
+
     def keypress(self, size, key):
         if self.walker.editing:
             if key in ["esc"]:
@@ -317,13 +408,18 @@ class GridEditor(urwid.WidgetWrap):
             return None
 
         key = common.shortcuts(key)
+        column = self.columns[self.walker.focus_col]
         if key in ["q", "esc"]:
             res = []
             for i in self.walker.lst:
-                if not i[1] and any([x.strip() for x in i[0]]):
+                if not i[1] and any([x for x in i[0]]):
                     res.append(i[0])
-            self.callback(res, *self.cb_args, **self.cb_kwargs)
+            self.callback(self.data_out(res), *self.cb_args, **self.cb_kwargs)
-            self.master.pop_view()
+            signals.pop_view_state.send(self)
+        elif key == "G":
+            self.walker.set_focus(0)
+        elif key == "g":
+            self.walker.set_focus(len(self.walker.lst) - 1)
         elif key in ["h", "left"]:
             self.walker.left()
         elif key in ["l", "right"]:
@@ -336,26 +432,22 @@ class GridEditor(urwid.WidgetWrap):
             self.walker.insert()
         elif key == "d":
             self.walker.delete_focus()
-        elif key == "r":
+        elif column.keypress(key, self) and not self.handle_key(key):
-            if self.walker.get_current_value() is not None:
-                self.master.path_prompt("Read file: ", "", self.read_file)
-        elif key == "R":
-            if self.walker.get_current_value() is not None:
-                self.master.path_prompt(
-                    "Read unescaped file: ", "", self.read_file, True
-                )
-        elif key == "e":
-            o = self.walker.get_current_value()
-            if o is not None:
-                n = self.master.spawn_editor(o.encode("string-escape"))
-                n = utils.clean_hanging_newline(n)
-                self.walker.set_current_value(n, False)
-                self.walker._modified()
-        elif key in ["enter"]:
-            self.walker.start_edit()
-        elif not self.handle_key(key):
             return self._w.keypress(size, key)
 
+    def data_out(self, data):
+        """
+            Called on raw list data, before data is returned through the
+            callback.
+        """
+        return data
+
+    def data_in(self, data):
+        """
+            Called to prepare provided data.
+        """
+        return data
+
     def is_error(self, col, val):
         """
             Return False, or a string error message.
@@ -373,10 +465,10 @@ class GridEditor(urwid.WidgetWrap):
             ("a", "add row after cursor"),
             ("d", "delete row"),
             ("e", "spawn external editor on current field"),
-            ("q", "return to flow view"),
+            ("q", "save changes and exit editor"),
             ("r", "read value from file"),
             ("R", "read unescaped value from file"),
-            ("esc", "return to flow view/exit field edit mode"),
+            ("esc", "save changes and exit editor"),
             ("tab", "next field"),
             ("enter", "edit field"),
         ]
@@ -396,14 +488,18 @@ class GridEditor(urwid.WidgetWrap):
 
 class QueryEditor(GridEditor):
     title = "Editing query"
-    columns = 2
+    columns = [
-    headings = ("Key", "Value")
+        TextColumn("Key"),
+        TextColumn("Value")
+    ]
 
 
 class HeaderEditor(GridEditor):
     title = "Editing headers"
-    columns = 2
+    columns = [
-    headings = ("Key", "Value")
+        TextColumn("Key"),
+        TextColumn("Value")
+    ]
 
     def make_help(self):
         h = GridEditor.make_help(self)
@@ -431,24 +527,29 @@ class HeaderEditor(GridEditor):
 
     def handle_key(self, key):
         if key == "U":
-            self.master.prompt_onekey(
+            signals.status_prompt_onekey.send(
-                "Add User-Agent header:",
+                prompt = "Add User-Agent header:",
-                [(i[0], i[1]) for i in http_uastrings.UASTRINGS],
+                keys = [(i[0], i[1]) for i in http_uastrings.UASTRINGS],
-                self.set_user_agent,
+                callback = self.set_user_agent,
             )
             return True
 
 
 class URLEncodedFormEditor(GridEditor):
     title = "Editing URL-encoded form"
-    columns = 2
+    columns = [
-    headings = ("Key", "Value")
+        TextColumn("Key"),
+        TextColumn("Value")
+    ]
 
 
 class ReplaceEditor(GridEditor):
     title = "Editing replacement patterns"
-    columns = 3
+    columns = [
-    headings = ("Filter", "Regex", "Replacement")
+        TextColumn("Filter"),
+        TextColumn("Regex"),
+        TextColumn("Replacement"),
+    ]
 
     def is_error(self, col, val):
         if col == 0:
@@ -464,8 +565,11 @@ class ReplaceEditor(GridEditor):
 
 class SetHeadersEditor(GridEditor):
     title = "Editing header set patterns"
-    columns = 3
+    columns = [
-    headings = ("Filter", "Header", "Value")
+        TextColumn("Filter"),
+        TextColumn("Header"),
+        TextColumn("Value"),
+    ]
 
     def is_error(self, col, val):
         if col == 0:
@@ -500,39 +604,105 @@ class SetHeadersEditor(GridEditor):
 
     def handle_key(self, key):
         if key == "U":
-            self.master.prompt_onekey(
+            signals.status_prompt_onekey.send(
-                "Add User-Agent header:",
+                prompt = "Add User-Agent header:",
-                [(i[0], i[1]) for i in http_uastrings.UASTRINGS],
+                keys = [(i[0], i[1]) for i in http_uastrings.UASTRINGS],
-                self.set_user_agent,
+                callback = self.set_user_agent,
             )
             return True
 
 
 class PathEditor(GridEditor):
     title = "Editing URL path components"
-    columns = 1
+    columns = [
-    headings = ("Component",)
+        TextColumn("Component"),
+    ]
+
+    def data_in(self, data):
+        return [[i] for i in data]
+
+    def data_out(self, data):
+        return [i[0] for i in data]
 
 
 class ScriptEditor(GridEditor):
     title = "Editing scripts"
-    columns = 1
+    columns = [
-    headings = ("Command",)
+        TextColumn("Command"),
+    ]
 
     def is_error(self, col, val):
         try:
             script.Script.parse_command(val)
-        except script.ScriptError, v:
+        except script.ScriptError as v:
             return str(v)
 
 
 class HostPatternEditor(GridEditor):
     title = "Editing host patterns"
-    columns = 1
+    columns = [
-    headings = ("Regex (matched on hostname:port / ip:port)",)
+        TextColumn("Regex (matched on hostname:port / ip:port)")
+    ]
 
     def is_error(self, col, val):
         try:
             re.compile(val, re.IGNORECASE)
         except re.error as e:
             return "Invalid regex: %s" % str(e)
+
+    def data_in(self, data):
+        return [[i] for i in data]
+
+    def data_out(self, data):
+        return [i[0] for i in data]
+
+
+class CookieEditor(GridEditor):
+    title = "Editing request Cookie header"
+    columns = [
+        TextColumn("Name"),
+        TextColumn("Value"),
+    ]
+
+
+class CookieAttributeEditor(GridEditor):
+    title = "Editing Set-Cookie attributes"
+    columns = [
+        TextColumn("Name"),
+        TextColumn("Value"),
+    ]
+
+    def data_out(self, data):
+        ret = []
+        for i in data:
+            if not i[1]:
+                ret.append([i[0], None])
+            else:
+                ret.append(i)
+        return ret
+
+
+class SetCookieEditor(GridEditor):
+    title = "Editing response SetCookie header"
+    columns = [
+        TextColumn("Name"),
+        TextColumn("Value"),
+        SubgridColumn("Attributes", CookieAttributeEditor),
+    ]
+
+    def data_in(self, data):
+        flattened = []
+        for k, v in data.items():
+            flattened.append([k, v[0], v[1].lst])
+        return flattened
+
+    def data_out(self, data):
+        vals = []
+        for i in data:
+            vals.append(
+                [
+                    i[0],
+                    [i[1], odict.ODictCaseless(i[2])]
+                ]
+            )
+        return odict.ODict(vals)

libmproxy/console/help.py

@@ -2,18 +2,17 @@ from __future__ import absolute_import
 
 import urwid
 
-from . import common
+from . import common, signals
 from .. import filt, version
 
 footer = [
-    ("heading", 'mitmproxy v%s '%version.VERSION),
+    ("heading", 'mitmproxy v%s ' % version.VERSION),
     ('heading_key', "q"), ":back ",
 ]
 
 
 class HelpView(urwid.ListBox):
-    def __init__(self, master, help_context, state):
+    def __init__(self, help_context):
-        self.master, self.state = master, state
         self.help_context = help_context or []
         urwid.ListBox.__init__(
             self,
@@ -29,101 +28,26 @@ class HelpView(urwid.ListBox):
         keys = [
             ("j, k", "down, up"),
             ("h, l", "left, right (in some contexts)"),
+            ("g, G", "go to end, beginning"),
             ("space", "page down"),
             ("pg up/down", "page up/down"),
             ("arrows", "up, down, left, right"),
         ]
-        text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
+        text.extend(
+            common.format_keyvals(
+                keys,
+                key="key",
+                val="text",
+                indent=4))
 
         text.append(urwid.Text([("head", "\n\nGlobal keys:\n")]))
         keys = [
             ("c", "client replay"),
-            ("H", "edit global header set patterns"),
-            ("I", "set ignore pattern"),
             ("i", "set interception pattern"),
-            ("M", "change global default display mode"),
+            ("o", "options"),
-                (None,
+            ("q", "quit / return to previous page"),
-                    common.highlight_key("automatic", "a") +
-                    [("text", ": automatic detection")]
-                ),
-                (None,
-                    common.highlight_key("hex", "e") +
-                    [("text", ": Hex")]
-                ),
-                (None,
-                    common.highlight_key("html", "h") +
-                    [("text", ": HTML")]
-                ),
-                (None,
-                    common.highlight_key("image", "i") +
-                    [("text", ": Image")]
-                ),
-                (None,
-                    common.highlight_key("javascript", "j") +
-                    [("text", ": JavaScript")]
-                ),
-                (None,
-                    common.highlight_key("json", "s") +
-                    [("text", ": JSON")]
-                ),
-                (None,
-                    common.highlight_key("css", "c") +
-                    [("text", ": CSS")]
-                ),
-                (None,
-                    common.highlight_key("urlencoded", "u") +
-                    [("text", ": URL-encoded data")]
-                ),
-                (None,
-                    common.highlight_key("raw", "r") +
-                    [("text", ": raw data")]
-                ),
-                (None,
-                    common.highlight_key("xml", "x") +
-                    [("text", ": XML")]
-                ),
-                (None,
-                    common.highlight_key("wbxml", "w") +
-                    [("text", ": WBXML")]
-                ),
-                (None,
-                    common.highlight_key("amf", "f") +
-                    [("text", ": AMF (requires PyAMF)")]
-                ),
-            ("o", "toggle options:"),
-                (None,
-                    common.highlight_key("anticache", "a") +
-                    [("text", ": prevent cached responses")]
-                ),
-                (None,
-                    common.highlight_key("anticomp", "c") +
-                    [("text", ": prevent compressed responses")]
-                ),
-                (None,
-                    common.highlight_key("showhost", "h") +
-                    [("text", ": use Host header for URL display")]
-                ),
-                (None,
-                    common.highlight_key("killextra", "k") +
-                    [("text", ": kill requests not part of server replay")]
-                ),
-                (None,
-                    common.highlight_key("norefresh", "n") +
-                    [("text", ": disable server replay response refresh")]
-                ),
-                (None,
-                    common.highlight_key("upstream certs", "u") +
-                    [("text", ": sniff cert info from upstream server")]
-                ),
-
-            ("q", "quit / return to flow list"),
             ("Q", "quit without confirm prompt"),
-            ("R", "edit replacement patterns"),
-            ("s", "add/remove scripts"),
             ("S", "server replay"),
-            ("t", "set sticky cookie expression"),
-            ("T", "set tcp proxying pattern"),
-            ("u", "set sticky auth expression"),
         ]
         text.extend(
             common.format_keyvals(keys, key="key", val="text", indent=4)
@@ -133,15 +57,15 @@ class HelpView(urwid.ListBox):
         f = []
         for i in filt.filt_unary:
             f.append(
-                ("~%s"%i.code, i.help)
+                ("~%s" % i.code, i.help)
             )
         for i in filt.filt_rex:
             f.append(
-                ("~%s regex"%i.code, i.help)
+                ("~%s regex" % i.code, i.help)
             )
         for i in filt.filt_int:
             f.append(
-                ("~%s int"%i.code, i.help)
+                ("~%s int" % i.code, i.help)
             )
         f.sort()
         f.extend(
@@ -180,11 +104,12 @@ class HelpView(urwid.ListBox):
     def keypress(self, size, key):
         key = common.shortcuts(key)
         if key == "q":
-            self.master.statusbar = self.state[0]
+            signals.pop_view_state.send(self)
-            self.master.body = self.state[1]
-            self.master.header = self.state[2]
-            self.master.loop.widget = self.master.make_view()
             return None
         elif key == "?":
             key = None
+        elif key == "G":
+            self.set_focus(0)
+        elif key == "g":
+            self.set_focus(len(self.body.contents))
         return urwid.ListBox.keypress(self, size, key)

libmproxy/console/options.py

@@ -0,0 +1,269 @@
+import urwid
+
+from . import common, signals, grideditor, contentview
+from . import select, palettes
+
+footer = [
+    ('heading_key', "enter/space"), ":toggle ",
+    ('heading_key', "C"), ":clear all ",
+]
+
+
+def _mkhelp():
+    text = []
+    keys = [
+        ("enter/space", "activate option"),
+        ("C", "clear all options"),
+    ]
+    text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
+    return text
+help_context = _mkhelp()
+
+
+class Options(urwid.WidgetWrap):
+    def __init__(self, master):
+        self.master = master
+        self.lb = select.Select(
+            [
+                select.Heading("Traffic Manipulation"),
+                select.Option(
+                    "Header Set Patterns",
+                    "H",
+                    lambda: master.setheaders.count(),
+                    self.setheaders
+                ),
+                select.Option(
+                    "Ignore Patterns",
+                    "I",
+                    lambda: master.server.config.check_ignore,
+                    self.ignorepatterns
+                ),
+                select.Option(
+                    "Replacement Patterns",
+                    "R",
+                    lambda: master.replacehooks.count(),
+                    self.replacepatterns
+                ),
+                select.Option(
+                    "Scripts",
+                    "S",
+                    lambda: master.scripts,
+                    self.scripts
+                ),
+
+                select.Heading("Interface"),
+                select.Option(
+                    "Default Display Mode",
+                    "M",
+                    self.has_default_displaymode,
+                    self.default_displaymode
+                ),
+                select.Option(
+                    "Palette",
+                    "P",
+                    lambda: self.master.palette != palettes.DEFAULT,
+                    self.palette
+                ),
+                select.Option(
+                    "Show Host",
+                    "w",
+                    lambda: master.showhost,
+                    self.toggle_showhost
+                ),
+
+                select.Heading("Network"),
+                select.Option(
+                    "No Upstream Certs",
+                    "U",
+                    lambda: master.server.config.no_upstream_cert,
+                    self.toggle_upstream_cert
+                ),
+                select.Option(
+                    "TCP Proxying",
+                    "T",
+                    lambda: master.server.config.check_tcp,
+                    self.tcp_proxy
+                ),
+
+                select.Heading("Utility"),
+                select.Option(
+                    "Anti-Cache",
+                    "a",
+                    lambda: master.anticache,
+                    self.toggle_anticache
+                ),
+                select.Option(
+                    "Anti-Compression",
+                    "o",
+                    lambda: master.anticomp,
+                    self.toggle_anticomp
+                ),
+                select.Option(
+                    "Kill Extra",
+                    "x",
+                    lambda: master.killextra,
+                    self.toggle_killextra
+                ),
+                select.Option(
+                    "No Refresh",
+                    "f",
+                    lambda: not master.refresh_server_playback,
+                    self.toggle_refresh_server_playback
+                ),
+                select.Option(
+                    "Sticky Auth",
+                    "A",
+                    lambda: master.stickyauth_txt,
+                    self.sticky_auth
+                ),
+                select.Option(
+                    "Sticky Cookies",
+                    "t",
+                    lambda: master.stickycookie_txt,
+                    self.sticky_cookie
+                ),
+            ]
+        )
+        title = urwid.Text("Options")
+        title = urwid.Padding(title, align="left", width=("relative", 100))
+        title = urwid.AttrWrap(title, "heading")
+        self._w = urwid.Frame(
+            self.lb,
+            header = title
+        )
+        self.master.loop.widget.footer.update("")
+        signals.update_settings.connect(self.sig_update_settings)
+
+    def sig_update_settings(self, sender):
+        self.lb.walker._modified()
+
+    def keypress(self, size, key):
+        if key == "C":
+            self.clearall()
+            return None
+        return super(self.__class__, self).keypress(size, key)
+
+    def clearall(self):
+        self.master.anticache = False
+        self.master.anticomp = False
+        self.master.killextra = False
+        self.master.showhost = False
+        self.master.refresh_server_playback = True
+        self.master.server.config.no_upstream_cert = False
+        self.master.setheaders.clear()
+        self.master.replacehooks.clear()
+        self.master.set_ignore_filter([])
+        self.master.set_tcp_filter([])
+        self.master.scripts = []
+        self.master.set_stickyauth(None)
+        self.master.set_stickycookie(None)
+        self.master.state.default_body_view = contentview.get("Auto")
+
+        signals.update_settings.send(self)
+        signals.status_message.send(
+            message = "All select.Options cleared",
+            expire = 1
+        )
+
+    def toggle_anticache(self):
+        self.master.anticache = not self.master.anticache
+
+    def toggle_anticomp(self):
+        self.master.anticomp = not self.master.anticomp
+
+    def toggle_killextra(self):
+        self.master.killextra = not self.master.killextra
+
+    def toggle_showhost(self):
+        self.master.showhost = not self.master.showhost
+
+    def toggle_refresh_server_playback(self):
+        self.master.refresh_server_playback = not self.master.refresh_server_playback
+
+    def toggle_upstream_cert(self):
+        self.master.server.config.no_upstream_cert = not self.master.server.config.no_upstream_cert
+        signals.update_settings.send(self)
+
+    def setheaders(self):
+        def _set(*args, **kwargs):
+            self.master.setheaders.set(*args, **kwargs)
+            signals.update_settings.send(self)
+        self.master.view_grideditor(
+            grideditor.SetHeadersEditor(
+                self.master,
+                self.master.setheaders.get_specs(),
+                _set
+            )
+        )
+
+    def ignorepatterns(self):
+        def _set(ignore):
+            self.master.set_ignore_filter(ignore)
+            signals.update_settings.send(self)
+        self.master.view_grideditor(
+            grideditor.HostPatternEditor(
+                self.master,
+                self.master.get_ignore_filter(),
+                _set
+            )
+        )
+
+    def replacepatterns(self):
+        def _set(*args, **kwargs):
+            self.master.replacehooks.set(*args, **kwargs)
+            signals.update_settings.send(self)
+        self.master.view_grideditor(
+            grideditor.ReplaceEditor(
+                self.master,
+                self.master.replacehooks.get_specs(),
+                _set
+            )
+        )
+
+    def scripts(self):
+        self.master.view_grideditor(
+            grideditor.ScriptEditor(
+                self.master,
+                [[i.command] for i in self.master.scripts],
+                self.master.edit_scripts
+            )
+        )
+
+    def default_displaymode(self):
+        signals.status_prompt_onekey.send(
+            prompt = "Global default display mode",
+            keys = contentview.view_prompts,
+            callback = self.master.change_default_display_mode
+        )
+
+    def has_default_displaymode(self):
+        return self.master.state.default_body_view.name != "Auto"
+
+    def tcp_proxy(self):
+        def _set(tcp):
+            self.master.set_tcp_filter(tcp)
+            signals.update_settings.send(self)
+        self.master.view_grideditor(
+            grideditor.HostPatternEditor(
+                self.master,
+                self.master.get_tcp_filter(),
+                _set
+            )
+        )
+
+    def sticky_auth(self):
+        signals.status_prompt.send(
+            prompt = "Sticky auth filter",
+            text = self.master.stickyauth_txt,
+            callback = self.master.set_stickyauth
+        )
+
+    def sticky_cookie(self):
+        signals.status_prompt.send(
+            prompt = "Sticky cookie filter",
+            text = self.master.stickycookie_txt,
+            callback = self.master.set_stickycookie
+        )
+
+    def palette(self):
+        self.master.view_palette_picker()

libmproxy/console/palettepicker.py

@@ -0,0 +1,81 @@
+import urwid
+
+from . import select, common, palettes, signals
+
+footer = [
+    ('heading_key', "enter/space"), ":select",
+]
+
+
+def _mkhelp():
+    text = []
+    keys = [
+        ("enter/space", "select"),
+    ]
+    text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
+    return text
+help_context = _mkhelp()
+
+
+class PalettePicker(urwid.WidgetWrap):
+    def __init__(self, master):
+        self.master = master
+        low, high = [], []
+        for k, v in palettes.palettes.items():
+            if v.high:
+                high.append(k)
+            else:
+                low.append(k)
+        high.sort()
+        low.sort()
+
+        options = [
+            select.Heading("High Colour")
+        ]
+
+        def mkopt(name):
+            return select.Option(
+                i,
+                None,
+                lambda: self.master.palette == name,
+                lambda: self.select(name)
+            )
+
+        for i in high:
+            options.append(mkopt(i))
+        options.append(select.Heading("Low Colour"))
+        for i in low:
+            options.append(mkopt(i))
+
+        options.extend(
+            [
+                select.Heading("Options"),
+                select.Option(
+                    "Transparent",
+                    "T",
+                    lambda: master.palette_transparent,
+                    self.toggle_palette_transparent
+                )
+            ]
+        )
+
+        self.lb = select.Select(options)
+        title = urwid.Text("Palettes")
+        title = urwid.Padding(title, align="left", width=("relative", 100))
+        title = urwid.AttrWrap(title, "heading")
+        self._w = urwid.Frame(
+            self.lb,
+            header = title
+        )
+        signals.update_settings.connect(self.sig_update_settings)
+
+    def sig_update_settings(self, sender):
+        self.lb.walker._modified()
+
+    def select(self, name):
+        self.master.set_palette(name)
+
+    def toggle_palette_transparent(self):
+        self.master.palette_transparent = not self.master.palette_transparent
+        self.master.set_palette(self.master.palette)
+        signals.update_settings.send(self)

libmproxy/console/palettes.py

@@ -1,4 +1,3 @@
-
 # Low-color themes should ONLY use the standard foreground and background
 # colours listed here:
 #
@@ -6,9 +5,9 @@
 #
 
 
-
 class Palette:
     _fields = [
+        'background',
         'title',
 
         # Status bar & heading
@@ -17,6 +16,10 @@ class Palette:
         # Help
         'key', 'head', 'text',
 
+        # Options
+        'option_selected', 'option_active', 'option_active_selected',
+        'option_selected_key',
+
         # List and Connections
         'method', 'focus',
         'code_200', 'code_300', 'code_400', 'code_500', 'code_other',
@@ -31,14 +34,32 @@ class Palette:
     ]
     high = None
 
-    def palette(self):
+    def palette(self, transparent):
         l = []
+        highback, lowback = None, None
+        if not transparent:
+            if self.high and self.high.get("background"):
+                highback = self.high["background"][1]
+            lowback = self.low["background"][1]
+
         for i in self._fields:
+            if transparent and i == "background":
+                l.append(["background", "default", "default"])
+            else:
                 v = [i]
-            v.extend(self.low[i])
+                low = list(self.low[i])
+                if lowback and low[1] == "default":
+                    low[1] = lowback
+                v.extend(low)
                 if self.high and i in self.high:
                     v.append(None)
-                v.extend(self.high[i])
+                    high = list(self.high[i])
+                    if highback and high[1] == "default":
+                        high[1] = highback
+                    v.extend(high)
+                elif highback and self.low[i][1] == "default":
+                    high = [None, low[0], highback]
+                    v.extend(high)
                 l.append(tuple(v))
         return l
 
@@ -48,18 +69,25 @@ class LowDark(Palette):
         Low-color dark background
     """
     low = dict(
+        background = ('white', 'black'),
         title = ('white,bold', 'default'),
 
         # Status bar & heading
-        heading = ('light gray', 'dark blue'),
+        heading = ('white', 'dark blue'),
         heading_key = ('light cyan', 'dark blue'),
-        heading_inactive = ('white', 'dark gray'),
+        heading_inactive = ('dark gray', 'light gray'),
 
         # Help
         key = ('light cyan', 'default'),
         head = ('white,bold', 'default'),
         text = ('light gray', 'default'),
 
+        # Options
+        option_selected = ('black', 'light gray'),
+        option_selected_key = ('light cyan', 'light gray'),
+        option_active = ('light red', 'default'),
+        option_active_selected = ('light red', 'light gray'),
+
         # List and Connections
         method = ('dark cyan', 'default'),
         focus = ('yellow', 'default'),
@@ -92,6 +120,10 @@ class Dark(LowDark):
     high = dict(
         heading_inactive = ('g58', 'g11'),
         intercept = ('#f60', 'default'),
+
+        option_selected = ('g85', 'g45'),
+        option_selected_key = ('light cyan', 'g50'),
+        option_active_selected = ('light red', 'g50'),
     )
 
 
@@ -100,18 +132,25 @@ class LowLight(Palette):
         Low-color light background
     """
     low = dict(
-        title = ('dark magenta,bold', 'light blue'),
+        background = ('black', 'white'),
+        title = ('dark magenta', 'default'),
 
         # Status bar & heading
-        heading = ('light gray', 'dark blue'),
+        heading = ('white', 'black'),
-        heading_key = ('light cyan', 'dark blue'),
+        heading_key = ('dark blue', 'black'),
         heading_inactive = ('black', 'light gray'),
 
         # Help
-        key = ('dark blue,bold', 'default'),
+        key = ('dark blue', 'default'),
-        head = ('black,bold', 'default'),
+        head = ('black', 'default'),
         text = ('dark gray', 'default'),
 
+        # Options
+        option_selected = ('black', 'light gray'),
+        option_selected_key = ('dark blue', 'light gray'),
+        option_active = ('light red', 'default'),
+        option_active_selected = ('light red', 'light gray'),
+
         # List and Connections
         method = ('dark cyan', 'default'),
         focus = ('black', 'default'),
@@ -142,10 +181,15 @@ class LowLight(Palette):
 
 class Light(LowLight):
     high = dict(
+        background = ('black', 'g100'),
         heading = ('g99', '#08f'),
         heading_key = ('#0ff,bold', '#08f'),
         heading_inactive = ('g35', 'g85'),
         replay = ('#0a0,bold', 'default'),
+
+        option_selected = ('black', 'g85'),
+        option_selected_key = ('dark blue', 'g85'),
+        option_active_selected = ('light red', 'g85'),
     )
 
 
@@ -167,9 +211,12 @@ sol_violet = "h61"
 sol_blue = "h33"
 sol_cyan = "h37"
 sol_green = "h64"
+
+
 class SolarizedLight(LowLight):
     high = dict(
-        title = (sol_blue, 'default'),
+        background = (sol_base00, sol_base3),
+        title = (sol_cyan, 'default'),
         text = (sol_base00, 'default'),
 
         # Status bar & heading
@@ -181,6 +228,12 @@ class SolarizedLight(LowLight):
         key = (sol_blue, 'default',),
         head = (sol_base00, 'default'),
 
+        # Options
+        option_selected = (sol_base03, sol_base2),
+        option_selected_key = (sol_blue, sol_base2),
+        option_active = (sol_orange, 'default'),
+        option_active_selected = (sol_orange, sol_base2),
+
         # List and Connections
         method = (sol_cyan, 'default'),
         focus = (sol_base01, 'default'),
@@ -193,7 +246,7 @@ class SolarizedLight(LowLight):
 
         error = (sol_red, 'default'),
 
-        header = (sol_base01, 'default'),
+        header = (sol_blue, 'default'),
         highlight = (sol_base01, 'default'),
         intercept = (sol_red, 'default',),
         replay = (sol_green, 'default',),
@@ -211,17 +264,24 @@ class SolarizedLight(LowLight):
 
 class SolarizedDark(LowDark):
     high = dict(
+        background = (sol_base2, sol_base03),
         title = (sol_blue, 'default'),
-        text = (sol_base0, 'default'),
+        text = (sol_base1, 'default'),
 
         # Status bar & heading
-        heading = (sol_base03, sol_base1),
+        heading = (sol_base2, sol_base01),
-        heading_key = (sol_blue+",bold", sol_base1),
+        heading_key = (sol_blue + ",bold", sol_base01),
         heading_inactive = (sol_base1, sol_base02),
 
         # Help
         key = (sol_blue, 'default',),
-        head = (sol_base00, 'default'),
+        head = (sol_base2, 'default'),
+
+        # Options
+        option_selected = (sol_base03, sol_base00),
+        option_selected_key = (sol_blue, sol_base00),
+        option_active = (sol_orange, 'default'),
+        option_active_selected = (sol_orange, sol_base00),
 
         # List and Connections
         method = (sol_cyan, 'default'),
@@ -235,7 +295,7 @@ class SolarizedDark(LowDark):
 
         error = (sol_red, 'default'),
 
-        header = (sol_base01, 'default'),
+        header = (sol_blue, 'default'),
         highlight = (sol_base01, 'default'),
         intercept = (sol_red, 'default',),
         replay = (sol_green, 'default',),
@@ -251,6 +311,7 @@ class SolarizedDark(LowDark):
     )
 
 
+DEFAULT = "dark"
 palettes = {
     "lowlight": LowLight(),
     "lowdark": LowDark(),

libmproxy/console/pathedit.py

@@ -0,0 +1,69 @@
+import glob
+import os.path
+
+import urwid
+
+
+class _PathCompleter:
+    def __init__(self, _testing=False):
+        """
+            _testing: disables reloading of the lookup table to make testing
+            possible.
+        """
+        self.lookup, self.offset = None, None
+        self.final = None
+        self._testing = _testing
+
+    def reset(self):
+        self.lookup = None
+        self.offset = -1
+
+    def complete(self, txt):
+        """
+            Returns the next completion for txt, or None if there is no
+            completion.
+        """
+        path = os.path.expanduser(txt)
+        if not self.lookup:
+            if not self._testing:
+                # Lookup is a set of (display value, actual value) tuples.
+                self.lookup = []
+                if os.path.isdir(path):
+                    files = glob.glob(os.path.join(path, "*"))
+                    prefix = txt
+                else:
+                    files = glob.glob(path + "*")
+                    prefix = os.path.dirname(txt)
+                prefix = prefix or "./"
+                for f in files:
+                    display = os.path.join(prefix, os.path.basename(f))
+                    if os.path.isdir(f):
+                        display += "/"
+                    self.lookup.append((display, f))
+            if not self.lookup:
+                self.final = path
+                return path
+            self.lookup.sort()
+            self.offset = -1
+            self.lookup.append((txt, txt))
+        self.offset += 1
+        if self.offset >= len(self.lookup):
+            self.offset = 0
+        ret = self.lookup[self.offset]
+        self.final = ret[1]
+        return ret[0]
+
+
+class PathEdit(urwid.Edit, _PathCompleter):
+    def __init__(self, *args, **kwargs):
+        urwid.Edit.__init__(self, *args, **kwargs)
+        _PathCompleter.__init__(self)
+
+    def keypress(self, size, key):
+        if key == "tab":
+            comp = self.complete(self.get_edit_text())
+            self.set_edit_text(comp)
+            self.set_edit_pos(len(comp))
+        else:
+            self.reset()
+        return urwid.Edit.keypress(self, size, key)

libmproxy/console/searchable.py

@@ -0,0 +1,91 @@
+import urwid
+
+from . import signals
+
+
+class Highlight(urwid.AttrMap):
+    def __init__(self, t):
+        urwid.AttrMap.__init__(
+            self,
+            urwid.Text(t.text),
+            "focusfield",
+        )
+        self.backup = t
+
+
+class Searchable(urwid.ListBox):
+    def __init__(self, state, contents):
+        self.walker = urwid.SimpleFocusListWalker(contents)
+        urwid.ListBox.__init__(self, self.walker)
+        self.state = state
+        self.search_offset = 0
+        self.current_highlight = None
+        self.search_term = None
+
+    def keypress(self, size, key):
+        if key == "/":
+            signals.status_prompt.send(
+                prompt = "Search for",
+                text = "",
+                callback = self.set_search
+            )
+        elif key == "n":
+            self.find_next(False)
+        elif key == "N":
+            self.find_next(True)
+        elif key == "G":
+            self.set_focus(0)
+            self.walker._modified()
+        elif key == "g":
+            self.set_focus(len(self.walker) - 1)
+            self.walker._modified()
+        else:
+            return super(self.__class__, self).keypress(size, key)
+
+    def set_search(self, text):
+        self.state.last_search = text
+        self.search_term = text or None
+        self.find_next(False)
+
+    def set_highlight(self, offset):
+        if self.current_highlight is not None:
+            old = self.body[self.current_highlight]
+            self.body[self.current_highlight] = old.backup
+        if offset is None:
+            self.current_highlight = None
+        else:
+            self.body[offset] = Highlight(self.body[offset])
+            self.current_highlight = offset
+
+    def get_text(self, w):
+        if isinstance(w, urwid.Text):
+            return w.text
+        elif isinstance(w, Highlight):
+            return w.backup.text
+        else:
+            return None
+
+    def find_next(self, backwards):
+        if not self.search_term:
+            if self.state.last_search:
+                self.search_term = self.state.last_search
+            else:
+                self.set_highlight(None)
+                return
+        # Start search at focus + 1
+        if backwards:
+            rng = xrange(len(self.body) - 1, -1, -1)
+        else:
+            rng = xrange(1, len(self.body) + 1)
+        for i in rng:
+            off = (self.focus_position + i) % len(self.body)
+            w = self.body[off]
+            txt = self.get_text(w)
+            if txt and self.search_term in txt:
+                self.set_highlight(off)
+                self.set_focus(off, coming_from="above")
+                self.body._modified()
+                return
+        else:
+            self.set_highlight(None)
+            signals.status_message.send(message="Search not found.", expire=1)

libmproxy/console/select.py

@@ -0,0 +1,115 @@
+import urwid
+
+from . import common
+
+
+class _OptionWidget(urwid.WidgetWrap):
+    def __init__(self, option, text, shortcut, active, focus):
+        self.option = option
+        textattr = "text"
+        keyattr = "key"
+        if focus and active:
+            textattr = "option_active_selected"
+            keyattr = "option_selected_key"
+        elif focus:
+            textattr = "option_selected"
+            keyattr = "option_selected_key"
+        elif active:
+            textattr = "option_active"
+        if shortcut:
+            text = common.highlight_key(
+                text,
+                shortcut,
+                textattr = textattr,
+                keyattr = keyattr
+            )
+        opt = urwid.Text(text, align="left")
+        opt = urwid.AttrWrap(opt, textattr)
+        opt = urwid.Padding(opt, align = "center", width = 40)
+        urwid.WidgetWrap.__init__(self, opt)
+
+    def keypress(self, size, key):
+        return key
+
+    def selectable(self):
+        return True
+
+
+class OptionWalker(urwid.ListWalker):
+    def __init__(self, options):
+        urwid.ListWalker.__init__(self)
+        self.options = options
+        self.focus = 0
+
+    def set_focus(self, pos):
+        self.focus = pos
+
+    def get_focus(self):
+        return self.options[self.focus].render(True), self.focus
+
+    def get_next(self, pos):
+        if pos >= len(self.options) - 1:
+            return None, None
+        return self.options[pos + 1].render(False), pos + 1
+
+    def get_prev(self, pos):
+        if pos <= 0:
+            return None, None
+        return self.options[pos - 1].render(False), pos - 1
+
+
+class Heading:
+    def __init__(self, text):
+        self.text = text
+
+    def render(self, focus):
+        opt = urwid.Text("\n" + self.text, align="left")
+        opt = urwid.AttrWrap(opt, "title")
+        opt = urwid.Padding(opt, align = "center", width = 40)
+        return opt
+
+
+_neg = lambda: False
+
+
+class Option:
+    def __init__(self, text, shortcut, getstate=None, activate=None):
+        self.text = text
+        self.shortcut = shortcut
+        self.getstate = getstate or _neg
+        self.activate = activate or _neg
+
+    def render(self, focus):
+        return _OptionWidget(
+            self,
+            self.text,
+            self.shortcut,
+            self.getstate(),
+            focus)
+
+
+class Select(urwid.ListBox):
+    def __init__(self, options):
+        self.walker = OptionWalker(options)
+        urwid.ListBox.__init__(
+            self,
+            self.walker
+        )
+        self.options = options
+        self.keymap = {}
+        for i in options:
+            if hasattr(i, "shortcut") and i.shortcut:
+                if i.shortcut in self.keymap:
+                    raise ValueError("Duplicate shortcut key: %s" % i.shortcut)
+                self.keymap[i.shortcut] = i
+
+    def keypress(self, size, key):
+        if key == "enter" or key == " ":
+            self.get_focus()[0].option.activate()
+            return None
+        key = common.shortcuts(key)
+        if key in self.keymap:
+            self.keymap[key].activate()
+            self.set_focus(self.options.index(self.keymap[key]))
+            return None
+        return super(self.__class__, self).keypress(size, key)

libmproxy/console/signals.py

@@ -0,0 +1,32 @@
+import blinker
+
+# Show a status message in the action bar
+status_message = blinker.Signal()
+
+# Prompt for input
+status_prompt = blinker.Signal()
+
+# Prompt for a path
+status_prompt_path = blinker.Signal()
+
+# Prompt for a single keystroke
+status_prompt_onekey = blinker.Signal()
+
+# Call a callback in N seconds
+call_in = blinker.Signal()
+
+# Focus the body, footer or header of the main window
+focus = blinker.Signal()
+
+# Fired when settings change
+update_settings = blinker.Signal()
+
+# Fired when a flow changes
+flow_change = blinker.Signal()
+
+# Fired when the flow list or focus changes
+flowlist_change = blinker.Signal()
+
+# Pop and push view state onto a stack
+pop_view_state = blinker.Signal()
+push_view_state = blinker.Signal()

libmproxy/console/statusbar.py

@@ -0,0 +1,254 @@
+import os.path
+
+import urwid
+
+import netlib.utils
+from . import pathedit, signals, common
+from .. import utils
+
+
+class ActionBar(urwid.WidgetWrap):
+    def __init__(self):
+        urwid.WidgetWrap.__init__(self, None)
+        self.clear()
+        signals.status_message.connect(self.sig_message)
+        signals.status_prompt.connect(self.sig_prompt)
+        signals.status_prompt_path.connect(self.sig_path_prompt)
+        signals.status_prompt_onekey.connect(self.sig_prompt_onekey)
+
+        self.last_path = ""
+
+        self.prompting = False
+        self.onekey = False
+        self.pathprompt = False
+
+    def sig_message(self, sender, message, expire=None):
+        w = urwid.Text(message)
+        self._w = w
+        if expire:
+            def cb(*args):
+                if w == self._w:
+                    self.clear()
+            signals.call_in.send(seconds=expire, callback=cb)
+
+    def prep_prompt(self, p):
+        return p.strip() + ": "
+
+    def sig_prompt(self, sender, prompt, text, callback, args=()):
+        signals.focus.send(self, section="footer")
+        self._w = urwid.Edit(self.prep_prompt(prompt), text or "")
+        self.prompting = (callback, args)
+
+    def sig_path_prompt(self, sender, prompt, callback, args=()):
+        signals.focus.send(self, section="footer")
+        self._w = pathedit.PathEdit(
+            self.prep_prompt(prompt),
+            os.path.dirname(self.last_path)
+        )
+        self.pathprompt = True
+        self.prompting = (callback, args)
+
+    def sig_prompt_onekey(self, sender, prompt, keys, callback, args=()):
+        """
+            Keys are a set of (word, key) tuples. The appropriate key in the
+            word is highlighted.
+        """
+        signals.focus.send(self, section="footer")
+        prompt = [prompt, " ("]
+        mkup = []
+        for i, e in enumerate(keys):
+            mkup.extend(common.highlight_key(e[0], e[1]))
+            if i < len(keys) - 1:
+                mkup.append(",")
+        prompt.extend(mkup)
+        prompt.append(")? ")
+        self.onekey = set(i[1] for i in keys)
+        self._w = urwid.Edit(prompt, "")
+        self.prompting = (callback, args)
+
+    def selectable(self):
+        return True
+
+    def keypress(self, size, k):
+        if self.prompting:
+            if k == "esc":
+                self.prompt_done()
+            elif self.onekey:
+                if k == "enter":
+                    self.prompt_done()
+                elif k in self.onekey:
+                    self.prompt_execute(k)
+            elif k == "enter":
+                self.prompt_execute(self._w.get_edit_text())
+            else:
+                if common.is_keypress(k):
+                    self._w.keypress(size, k)
+                else:
+                    return k
+
+    def clear(self):
+        self._w = urwid.Text("")
+
+    def prompt_done(self):
+        self.prompting = False
+        self.onekey = False
+        self.pathprompt = False
+        signals.status_message.send(message="")
+        signals.focus.send(self, section="body")
+
+    def prompt_execute(self, txt):
+        if self.pathprompt:
+            self.last_path = txt
+        p, args = self.prompting
+        self.prompt_done()
+        msg = p(txt, *args)
+        if msg:
+            signals.status_message.send(message=msg, expire=1)
+
+
+class StatusBar(urwid.WidgetWrap):
+    def __init__(self, master, helptext):
+        self.master, self.helptext = master, helptext
+        self.ab = ActionBar()
+        self.ib = urwid.WidgetWrap(urwid.Text(""))
+        self._w = urwid.Pile([self.ib, self.ab])
+        signals.update_settings.connect(self.sig_update_settings)
+        signals.flowlist_change.connect(self.sig_update_settings)
+        self.redraw()
+
+    def sig_update_settings(self, sender):
+        self.redraw()
+
+    def keypress(self, *args, **kwargs):
+        return self.ab.keypress(*args, **kwargs)
+
+    def get_status(self):
+        r = []
+
+        if self.master.setheaders.count():
+            r.append("[")
+            r.append(("heading_key", "H"))
+            r.append("eaders]")
+        if self.master.replacehooks.count():
+            r.append("[")
+            r.append(("heading_key", "R"))
+            r.append("eplacing]")
+        if self.master.client_playback:
+            r.append("[")
+            r.append(("heading_key", "cplayback"))
+            r.append(":%s to go]" % self.master.client_playback.count())
+        if self.master.server_playback:
+            r.append("[")
+            r.append(("heading_key", "splayback"))
+            if self.master.nopop:
+                r.append(":%s in file]" % self.master.server_playback.count())
+            else:
+                r.append(":%s to go]" % self.master.server_playback.count())
+        if self.master.get_ignore_filter():
+            r.append("[")
+            r.append(("heading_key", "I"))
+            r.append("gnore:%d]" % len(self.master.get_ignore_filter()))
+        if self.master.get_tcp_filter():
+            r.append("[")
+            r.append(("heading_key", "T"))
+            r.append("CP:%d]" % len(self.master.get_tcp_filter()))
+        if self.master.state.intercept_txt:
+            r.append("[")
+            r.append(("heading_key", "i"))
+            r.append(":%s]" % self.master.state.intercept_txt)
+        if self.master.state.limit_txt:
+            r.append("[")
+            r.append(("heading_key", "l"))
+            r.append(":%s]" % self.master.state.limit_txt)
+        if self.master.stickycookie_txt:
+            r.append("[")
+            r.append(("heading_key", "t"))
+            r.append(":%s]" % self.master.stickycookie_txt)
+        if self.master.stickyauth_txt:
+            r.append("[")
+            r.append(("heading_key", "u"))
+            r.append(":%s]" % self.master.stickyauth_txt)
+        if self.master.state.default_body_view.name != "Auto":
+            r.append("[")
+            r.append(("heading_key", "M"))
+            r.append(":%s]" % self.master.state.default_body_view.name)
+
+        opts = []
+        if self.master.anticache:
+            opts.append("anticache")
+        if self.master.anticomp:
+            opts.append("anticomp")
+        if self.master.showhost:
+            opts.append("showhost")
+        if not self.master.refresh_server_playback:
+            opts.append("norefresh")
+        if self.master.killextra:
+            opts.append("killextra")
+        if self.master.server.config.no_upstream_cert:
+            opts.append("no-upstream-cert")
+        if self.master.state.follow_focus:
+            opts.append("following")
+        if self.master.stream_large_bodies:
+            opts.append(
+                "stream:%s" % netlib.utils.pretty_size(
+                    self.master.stream_large_bodies.max_size
+                )
+            )
+
+        if opts:
+            r.append("[%s]" % (":".join(opts)))
+
+        if self.master.server.config.mode in ["reverse", "upstream"]:
+            dst = self.master.server.config.mode.dst
+            scheme = "https" if dst[0] else "http"
+            if dst[1] != dst[0]:
+                scheme += "2https" if dst[1] else "http"
+            r.append("[dest:%s]" % utils.unparse_url(scheme, *dst[2:]))
+        if self.master.scripts:
+            r.append("[")
+            r.append(("heading_key", "s"))
+            r.append("cripts:%s]" % len(self.master.scripts))
+        # r.append("[lt:%0.3f]"%self.master.looptime)
+
+        if self.master.stream:
+            r.append("[W:%s]" % self.master.stream_path)
+
+        return r
+
+    def redraw(self):
+        fc = self.master.state.flow_count()
+        if self.master.state.focus is None:
+            offset = 0
+        else:
+            offset = min(self.master.state.focus + 1, fc)
+        t = [
+            ('heading', ("[%s/%s]" % (offset, fc)).ljust(9))
+        ]
+
+        if self.master.server.bound:
+            host = self.master.server.address.host
+            if host == "0.0.0.0":
+                host = "*"
+            boundaddr = "[%s:%s]" % (host, self.master.server.address.port)
+        else:
+            boundaddr = ""
+        t.extend(self.get_status())
+        status = urwid.AttrWrap(urwid.Columns([
+            urwid.Text(t),
+            urwid.Text(
+                [
+                    self.helptext,
+                    boundaddr
+                ],
+                align="right"
+            ),
+        ]), "heading")
+        self.ib._w = status
+
+    def update(self, text):
+        self.helptext = text
+        self.redraw()
+        self.master.loop.draw_screen()
+
+    def selectable(self):
+        return True

libmproxy/console/tabs.py

@@ -0,0 +1,39 @@
+import urwid
+
+
+class Tabs(urwid.WidgetWrap):
+    def __init__(self, tabs, tab_offset=0):
+        urwid.WidgetWrap.__init__(self, "")
+        self.tab_offset = tab_offset
+        self.tabs = tabs
+        self.show()
+
+    def _tab(self, content, attr):
+        p = urwid.Text(content, align="center")
+        p = urwid.Padding(p, align="center", width=("relative", 100))
+        p = urwid.AttrWrap(p, attr)
+        return p
+
+    def keypress(self, size, key):
+        if key in ["tab", "l"]:
+            self.tab_offset = (self.tab_offset + 1) % (len(self.tabs))
+            self.show()
+        elif key == "h":
+            self.tab_offset = (self.tab_offset - 1) % (len(self.tabs))
+            self.show()
+        return self._w.keypress(size, key)
+
+    def show(self):
+        headers = []
+        for i in range(len(self.tabs)):
+            txt = self.tabs[i][0]()
+            if i == self.tab_offset:
+                headers.append(self._tab(txt, "heading"))
+            else:
+                headers.append(self._tab(txt, "heading_inactive"))
+        headers = urwid.Columns(headers, dividechars=1)
+        self._w = urwid.Frame(
+            body = self.tabs[self.tab_offset][1](),
+            header = headers
+        )
+        self._w.set_focus("body")

libmproxy/console/window.py

@@ -0,0 +1,72 @@
+import urwid
+from . import signals
+
+
+class Window(urwid.Frame):
+    def __init__(self, master, body, header, footer, helpctx):
+        urwid.Frame.__init__(
+            self,
+            urwid.AttrWrap(body, "background"),
+            header = urwid.AttrWrap(header, "background") if header else None,
+            footer = urwid.AttrWrap(footer, "background") if footer else None
+        )
+        self.master = master
+        self.helpctx = helpctx
+        signals.focus.connect(self.sig_focus)
+
+    def sig_focus(self, sender, section):
+        self.focus_position = section
+
+    def keypress(self, size, k):
+        k = super(self.__class__, self).keypress(size, k)
+        if k == "?":
+            self.master.view_help(self.helpctx)
+        elif k == "c":
+            if not self.master.client_playback:
+                signals.status_prompt_path.send(
+                    self,
+                    prompt = "Client replay",
+                    callback = self.master.client_playback_path
+                )
+            else:
+                signals.status_prompt_onekey.send(
+                    self,
+                    prompt = "Stop current client replay?",
+                    keys = (
+                        ("yes", "y"),
+                        ("no", "n"),
+                    ),
+                    callback = self.master.stop_client_playback_prompt,
+                )
+        elif k == "i":
+            signals.status_prompt.send(
+                self,
+                prompt = "Intercept filter",
+                text = self.master.state.intercept_txt,
+                callback = self.master.set_intercept
+            )
+        elif k == "o":
+            self.master.view_options()
+        elif k == "Q":
+            raise urwid.ExitMainLoop
+        elif k == "q":
+            signals.pop_view_state.send(self)
+        elif k == "S":
+            if not self.master.server_playback:
+                signals.status_prompt_path.send(
+                    self,
+                    prompt = "Server replay path",
+                    callback = self.master.server_playback_path
+                )
+            else:
+                signals.status_prompt_onekey.send(
+                    self,
+                    prompt = "Stop current server replay?",
+                    keys = (
+                        ("yes", "y"),
+                        ("no", "n"),
+                    ),
+                    callback = self.master.stop_server_playback_prompt,
+                )
+        else:
+            return k

libmproxy/controller.py

@@ -1,11 +1,14 @@
 from __future__ import absolute_import
-import Queue, threading
+import Queue
+import threading
+
 
 class DummyReply:
     """
         A reply object that does nothing. Useful when we need an object to seem
         like it has a channel, and during testing.
     """
+
     def __init__(self):
         self.acked = False
 
@@ -19,6 +22,7 @@ class Reply:
         This object is used to respond to the message through the return
         channel.
     """
+
     def __init__(self, obj):
         self.obj = obj
         self.q = Queue.Queue()
@@ -67,11 +71,13 @@ class Slave(threading.Thread):
         Slaves get a channel end-point through which they can send messages to
         the master.
     """
+
     def __init__(self, channel, server):
         self.channel, self.server = channel, server
         self.server.set_channel(channel)
         threading.Thread.__init__(self)
-        self.name = "SlaveThread (%s:%s)" % (self.server.address.host, self.server.address.port)
+        self.name = "SlaveThread (%s:%s)" % (
+            self.server.address.host, self.server.address.port)
 
     def run(self):
         self.server.serve_forever()
@@ -81,6 +87,7 @@ class Master(object):
     """
         Masters get and respond to messages from slaves.
     """
+
     def __init__(self, server):
         """
             server may be None if no server is needed.

libmproxy/dump.py

@@ -53,7 +53,7 @@ class Options(object):
 
 
 def str_response(resp):
-    r = "%s %s"%(resp.code, resp.msg)
+    r = "%s %s" % (resp.code, resp.msg)
     if resp.is_replay:
         r = "[replay] " + r
     return r
@@ -64,7 +64,7 @@ def str_request(f, showhost):
         c = f.client_conn.address.host
     else:
         c = "[replay]"
-    r = "%s %s %s"%(c, f.request.method, f.request.pretty_url(showhost))
+    r = "%s %s %s" % (c, f.request.method, f.request.pretty_url(showhost))
     if f.request.stickycookie:
         r = "[stickycookie] " + r
     return r
@@ -102,7 +102,7 @@ class DumpMaster(flow.FlowMaster):
             try:
                 f = file(path, options.outfile[1])
                 self.start_stream(f, self.filt)
-            except IOError, v:
+            except IOError as v:
                 raise DumpError(v.strerror)
 
         if options.replacements:
@@ -140,7 +140,7 @@ class DumpMaster(flow.FlowMaster):
         if options.rfile:
             try:
                 self.load_flows_file(options.rfile)
-            except flow.FlowReadError, v:
+            except flow.FlowReadError as v:
                 self.add_event("Flow file corrupted.", "error")
                 raise DumpError(v)
 
@@ -171,7 +171,7 @@ class DumpMaster(flow.FlowMaster):
 
     def _print_message(self, message):
         if self.o.flow_detail >= 2:
-            print(self.indent(4, message.headers), file=self.outfile)
+            print(self.indent(4, message.headers.format()), file=self.outfile)
         if self.o.flow_detail >= 3:
             if message.content == http.CONTENT_MISSING:
                 print(self.indent(4, "(content missing)"), file=self.outfile)
@@ -181,12 +181,18 @@ class DumpMaster(flow.FlowMaster):
                 if not utils.isBin(content):
                     try:
                         jsn = json.loads(content)
-                        print(self.indent(4, json.dumps(jsn, indent=2)), file=self.outfile)
+                        print(
+                            self.indent(
+                                4,
+                                json.dumps(
+                                    jsn,
+                                    indent=2)),
+                            file=self.outfile)
                     except ValueError:
                         print(self.indent(4, content), file=self.outfile)
                 else:
                     d = netlib.utils.hexdump(content)
-                    d = "\n".join("%s\t%s %s"%i for i in d)
+                    d = "\n".join("%s\t%s %s" % i for i in d)
                     print(self.indent(4, d), file=self.outfile)
         if self.o.flow_detail >= 2:
             print("", file=self.outfile)
@@ -207,8 +213,13 @@ class DumpMaster(flow.FlowMaster):
             if f.response.content == http.CONTENT_MISSING:
                 sz = "(content missing)"
             else:
-                sz = utils.pretty_size(len(f.response.content))
+                sz = netlib.utils.pretty_size(len(f.response.content))
-            print(" << %s %s" % (str_response(f.response), sz), file=self.outfile)
+            print(
+                " << %s %s" %
+                (str_response(
+                    f.response),
+                    sz),
+                file=self.outfile)
             self._print_message(f.response)
 
         if f.error:

libmproxy/encoding.py

@@ -3,12 +3,14 @@
 """
 from __future__ import absolute_import
 import cStringIO
-import gzip, zlib
+import gzip
+import zlib
 
 __ALL__ = ["ENCODINGS"]
 
 ENCODINGS = set(["identity", "gzip", "deflate"])
 
+
 def decode(e, content):
     encoding_map = {
         "identity": identity,
@@ -19,6 +21,7 @@ def decode(e, content):
         return None
     return encoding_map[e](content)
 
+
 def encode(e, content):
     encoding_map = {
         "identity": identity,
@@ -29,6 +32,7 @@ def encode(e, content):
         return None
     return encoding_map[e](content)
 
+
 def identity(content):
     """
         Returns content unchanged. Identity is the default value of
@@ -36,6 +40,7 @@ def identity(content):
     """
     return content
 
+
 def decode_gzip(content):
     gfile = gzip.GzipFile(fileobj=cStringIO.StringIO(content))
     try:
@@ -43,6 +48,7 @@ def decode_gzip(content):
     except (IOError, EOFError):
         return None
 
+
 def encode_gzip(content):
     s = cStringIO.StringIO()
     gf = gzip.GzipFile(fileobj=s, mode='wb')
@@ -50,6 +56,7 @@ def encode_gzip(content):
     gf.close()
     return s.getvalue()
 
+
 def decode_deflate(content):
     """
         Returns decompressed data for DEFLATE. Some servers may respond with
@@ -67,6 +74,7 @@ def decode_deflate(content):
     except zlib.error:
         return None
 
+
 def encode_deflate(content):
     """
         Returns compressed content, always including zlib header and checksum.

libmproxy/filt.py

@@ -32,16 +32,17 @@
         rex         Equivalent to ~u rex
 """
 from __future__ import absolute_import
-import re, sys
+import re
+import sys
 from .contrib import pyparsing as pp
 from .protocol.http import decoded
 
 
 class _Token:
     def dump(self, indent=0, fp=sys.stdout):
-        print >> fp, "\t"*indent, self.__class__.__name__,
+        print >> fp, "\t" * indent, self.__class__.__name__,
         if hasattr(self, "expr"):
-            print >> fp, "(%s)"%self.expr,
+            print >> fp, "(%s)" % self.expr,
         print >> fp
 
 
@@ -54,6 +55,7 @@ class _Action(_Token):
 class FErr(_Action):
     code = "e"
     help = "Match error"
+
     def __call__(self, f):
         return True if f.error else False
 
@@ -61,6 +63,7 @@ class FErr(_Action):
 class FReq(_Action):
     code = "q"
     help = "Match request with no response"
+
     def __call__(self, f):
         if not f.response:
             return True
@@ -69,6 +72,7 @@ class FReq(_Action):
 class FResp(_Action):
     code = "s"
     help = "Match response"
+
     def __call__(self, f):
         return True if f.response else False
 
@@ -79,7 +83,7 @@ class _Rex(_Action):
         try:
             self.re = re.compile(self.expr)
         except:
-            raise ValueError, "Cannot compile expression."
+            raise ValueError("Cannot compile expression.")
 
 
 def _check_content_type(expr, o):
@@ -100,6 +104,7 @@ class FAsset(_Action):
         "image/.*",
         "application/x-shockwave-flash"
     ]
+
     def __call__(self, f):
         if f.response:
             for i in self.ASSET_TYPES:
@@ -111,6 +116,7 @@ class FAsset(_Action):
 class FContentType(_Rex):
     code = "t"
     help = "Content-type header"
+
     def __call__(self, f):
         if _check_content_type(self.expr, f.request):
             return True
@@ -122,6 +128,7 @@ class FContentType(_Rex):
 class FRequestContentType(_Rex):
     code = "tq"
     help = "Request Content-Type header"
+
     def __call__(self, f):
         return _check_content_type(self.expr, f.request)
 
@@ -129,6 +136,7 @@ class FRequestContentType(_Rex):
 class FResponseContentType(_Rex):
     code = "ts"
     help = "Response Content-Type header"
+
     def __call__(self, f):
         if f.response:
             return _check_content_type(self.expr, f.response)
@@ -138,6 +146,7 @@ class FResponseContentType(_Rex):
 class FHead(_Rex):
     code = "h"
     help = "Header"
+
     def __call__(self, f):
         if f.request.headers.match_re(self.expr):
             return True
@@ -149,6 +158,7 @@ class FHead(_Rex):
 class FHeadRequest(_Rex):
     code = "hq"
     help = "Request header"
+
     def __call__(self, f):
         if f.request.headers.match_re(self.expr):
             return True
@@ -157,6 +167,7 @@ class FHeadRequest(_Rex):
 class FHeadResponse(_Rex):
     code = "hs"
     help = "Response header"
+
     def __call__(self, f):
         if f.response and f.response.headers.match_re(self.expr):
             return True
@@ -165,6 +176,7 @@ class FHeadResponse(_Rex):
 class FBod(_Rex):
     code = "b"
     help = "Body"
+
     def __call__(self, f):
         if f.request and f.request.content:
             with decoded(f.request):
@@ -180,6 +192,7 @@ class FBod(_Rex):
 class FBodRequest(_Rex):
     code = "bq"
     help = "Request body"
+
     def __call__(self, f):
         if f.request and f.request.content:
             with decoded(f.request):
@@ -190,6 +203,7 @@ class FBodRequest(_Rex):
 class FBodResponse(_Rex):
     code = "bs"
     help = "Response body"
+
     def __call__(self, f):
         if f.response and f.response.content:
             with decoded(f.response):
@@ -200,6 +214,7 @@ class FBodResponse(_Rex):
 class FMethod(_Rex):
     code = "m"
     help = "Method"
+
     def __call__(self, f):
         return bool(re.search(self.expr, f.request.method, re.IGNORECASE))
 
@@ -207,6 +222,7 @@ class FMethod(_Rex):
 class FDomain(_Rex):
     code = "d"
     help = "Domain"
+
     def __call__(self, f):
         return bool(re.search(self.expr, f.request.host, re.IGNORECASE))
 
@@ -215,6 +231,7 @@ class FUrl(_Rex):
     code = "u"
     help = "URL"
     # FUrl is special, because it can be "naked".
+
     @classmethod
     def make(klass, s, loc, toks):
         if len(toks) > 1:
@@ -233,6 +250,7 @@ class _Int(_Action):
 class FCode(_Int):
     code = "c"
     help = "HTTP response code"
+
     def __call__(self, f):
         if f.response and f.response.code == self.num:
             return True
@@ -243,9 +261,9 @@ class FAnd(_Token):
         self.lst = lst
 
     def dump(self, indent=0, fp=sys.stdout):
-        print >> fp, "\t"*indent, self.__class__.__name__
+        print >> fp, "\t" * indent, self.__class__.__name__
         for i in self.lst:
-            i.dump(indent+1, fp)
+            i.dump(indent + 1, fp)
 
     def __call__(self, f):
         return all(i(f) for i in self.lst)
@@ -256,9 +274,9 @@ class FOr(_Token):
         self.lst = lst
 
     def dump(self, indent=0, fp=sys.stdout):
-        print >> fp, "\t"*indent, self.__class__.__name__
+        print >> fp, "\t" * indent, self.__class__.__name__
         for i in self.lst:
-            i.dump(indent+1, fp)
+            i.dump(indent + 1, fp)
 
     def __call__(self, f):
         return any(i(f) for i in self.lst)
@@ -269,7 +287,7 @@ class FNot(_Token):
         self.itm = itm[0]
 
     def dump(self, indent=0, fp=sys.stdout):
-        print >> fp, "\t"*indent, self.__class__.__name__
+        print >> fp, "\t" * indent, self.__class__.__name__
         self.itm.dump(indent + 1, fp)
 
     def __call__(self, f):
@@ -299,12 +317,14 @@ filt_rex = [
 filt_int = [
     FCode
 ]
+
+
 def _make():
     # Order is important - multi-char expressions need to come before narrow
     # ones.
     parts = []
     for klass in filt_unary:
-        f = pp.Literal("~%s"%klass.code)
+        f = pp.Literal("~%s" % klass.code)
         f.setParseAction(klass.make)
         parts.append(f)
 
@@ -313,12 +333,12 @@ def _make():
         pp.QuotedString("\"", escChar='\\') |\
         pp.QuotedString("'", escChar='\\')
     for klass in filt_rex:
-        f = pp.Literal("~%s"%klass.code) + rex.copy()
+        f = pp.Literal("~%s" % klass.code) + rex.copy()
         f.setParseAction(klass.make)
         parts.append(f)
 
     for klass in filt_int:
-        f = pp.Literal("~%s"%klass.code) + pp.Word(pp.nums)
+        f = pp.Literal("~%s" % klass.code) + pp.Word(pp.nums)
         f.setParseAction(klass.make)
         parts.append(f)
 
@@ -328,14 +348,20 @@ def _make():
     parts.append(f)
 
     atom = pp.MatchFirst(parts)
-    expr = pp.operatorPrecedence(
+    expr = pp.operatorPrecedence(atom,
-                atom,
+                                 [(pp.Literal("!").suppress(),
-                [
+                                   1,
-                    (pp.Literal("!").suppress(), 1, pp.opAssoc.RIGHT, lambda x: FNot(*x)),
+                                   pp.opAssoc.RIGHT,
-                    (pp.Literal("&").suppress(), 2, pp.opAssoc.LEFT, lambda x: FAnd(*x)),
+                                   lambda x: FNot(*x)),
-                    (pp.Literal("|").suppress(), 2, pp.opAssoc.LEFT, lambda x: FOr(*x)),
+                                     (pp.Literal("&").suppress(),
-                ]
+                                      2,
-           )
+                                      pp.opAssoc.LEFT,
+                                      lambda x: FAnd(*x)),
+                                     (pp.Literal("|").suppress(),
+                                      2,
+                                      pp.opAssoc.LEFT,
+                                      lambda x: FOr(*x)),
+                                  ])
     expr = pp.OneOrMore(expr)
     return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
 bnf = _make()
@@ -355,15 +381,15 @@ def parse(s):
 help = []
 for i in filt_unary:
     help.append(
-        ("~%s"%i.code, i.help)
+        ("~%s" % i.code, i.help)
     )
 for i in filt_rex:
     help.append(
-        ("~%s regex"%i.code, i.help)
+        ("~%s regex" % i.code, i.help)
     )
 for i in filt_int:
     help.append(
-        ("~%s int"%i.code, i.help)
+        ("~%s int" % i.code, i.help)
     )
 help.sort()
 help.extend(

libmproxy/flow.py

@@ -17,9 +17,6 @@ from .proxy.config import HostMatcher
 from .proxy.connection import ClientConnection, ServerConnection
 import urlparse
 
-ODict = odict.ODict
-ODictCaseless = odict.ODictCaseless
-
 
 class AppRegistry:
     def __init__(self):
@@ -165,7 +162,8 @@ class StreamLargeBodies(object):
             r.headers, is_request, flow.request.method, code
         )
         if not (0 <= expected_size <= self.max_size):
-            r.stream = r.stream or True  # r.stream may already be a callable, which we want to preserve.
+            # r.stream may already be a callable, which we want to preserve.
+            r.stream = r.stream or True
 
 
 class ClientPlaybackState:
@@ -203,8 +201,16 @@ class ClientPlaybackState:
 
 
 class ServerPlaybackState:
-    def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content,
+    def __init__(
-                 ignore_payload_params, ignore_host):
+            self,
+            headers,
+            flows,
+            exit,
+            nopop,
+            ignore_params,
+            ignore_content,
+            ignore_payload_params,
+            ignore_host):
         """
             headers: Case-insensitive list of request headers that should be
             included in request-response matching.
@@ -232,7 +238,7 @@ class ServerPlaybackState:
         r = flow.request
 
         _, _, path, _, query, _ = urlparse.urlparse(r.url)
-        queriesArray = urlparse.parse_qsl(query)
+        queriesArray = urlparse.parse_qsl(query, keep_blank_values=True)
 
         key = [
             str(r.port),
@@ -242,7 +248,7 @@ class ServerPlaybackState:
         ]
 
         if not self.ignore_content:
-            form_contents = r.get_form_urlencoded()
+            form_contents = r.get_form()
             if self.ignore_payload_params and form_contents:
                 key.extend(
                     p for p in form_contents
@@ -271,7 +277,7 @@ class ServerPlaybackState:
                 # to prevent a mismatch between unicode/non-unicode.
                 v = [str(x) for x in v]
                 hdrs.append((i, v))
-            key.append(repr(hdrs))
+            key.append(hdrs)
         return hashlib.sha256(repr(key)).digest()
 
     def next_flow(self, request):
@@ -462,6 +468,7 @@ class FlowStore(FlowList):
         Notifies the state that a flow has been updated.
         The flow must be present in the state.
         """
+        if f in self:
             for view in self.views:
                 view._update(f)
 
@@ -534,7 +541,8 @@ class State(object):
     def flow_count(self):
         return len(self.flows)
 
-    # TODO: All functions regarding flows that don't cause side-effects should be moved into FlowStore.
+    # TODO: All functions regarding flows that don't cause side-effects should
+    # be moved into FlowStore.
     def index(self, f):
         return self.flows.index(f)
 
@@ -593,6 +601,10 @@ class State(object):
     def accept_all(self, master):
         self.flows.accept_all(master)
 
+    def backup(self, f):
+        f.backup()
+        self.update_flow(f)
+
     def revert(self, f):
         f.revert()
         self.update_flow(f)
@@ -658,7 +670,7 @@ class FlowMaster(controller.Master):
         """
         try:
             s = script.Script(command, self)
-        except script.ScriptError, v:
+        except script.ScriptError as v:
             return v.args[0]
         self.scripts.append(s)
 
@@ -722,8 +734,17 @@ class FlowMaster(controller.Master):
     def stop_client_playback(self):
         self.client_playback = None
 
-    def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params,
+    def start_server_playback(
-                              ignore_content, ignore_payload_params, ignore_host):
+            self,
+            flows,
+            kill,
+            headers,
+            exit,
+            nopop,
+            ignore_params,
+            ignore_content,
+            ignore_payload_params,
+            ignore_host):
         """
             flows: List of flows.
             kill: Boolean, should we kill requests not part of the replay?
@@ -732,9 +753,15 @@ class FlowMaster(controller.Master):
             ignore_payload_params: list of content params to ignore in server replay
             ignore_host: true if request host should be ignored in server replay
         """
-        self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, 
+        self.server_playback = ServerPlaybackState(
-                                                   ignore_params, ignore_content,
+            headers,
-                                                   ignore_payload_params, ignore_host)
+            flows,
+            exit,
+            nopop,
+            ignore_params,
+            ignore_content,
+            ignore_payload_params,
+            ignore_host)
         self.kill_nonreplay = kill
 
     def stop_server_playback(self):
@@ -771,6 +798,8 @@ class FlowMaster(controller.Master):
             if all(e):
                 self.shutdown()
             self.client_playback.tick(self)
+            if self.client_playback.done():
+                self.client_playback = None
 
         return super(FlowMaster, self).tick(q, timeout)
 
@@ -789,16 +818,29 @@ class FlowMaster(controller.Master):
         s = ServerConnection.from_state(dict(
             address=dict(address=(host, port), use_ipv6=False),
             state=[],
-                source_address=None, #source_address=dict(address=(host, port), use_ipv6=False),
+            source_address=None,
+            # source_address=dict(address=(host, port), use_ipv6=False),
             cert=None,
             sni=host,
             ssl_established=True
         ))
-        f = http.HTTPFlow(c,s);
+        f = http.HTTPFlow(c, s)
         headers = ODictCaseless()
 
-        req = http.HTTPRequest("absolute", method, scheme, host, port, path, (1, 1), headers, None,
+        req = http.HTTPRequest(
-                                 None, None, None)
+            "absolute",
+            method,
+            scheme,
+            host,
+            port,
+            path,
+            (1,
+             1),
+            headers,
+            None,
+            None,
+            None,
+            None)
         f.request = req
         return self.load_flow(f)
 
@@ -809,7 +851,8 @@ class FlowMaster(controller.Master):
 
         if self.server and self.server.config.mode == "reverse":
             f.request.host, f.request.port = self.server.config.mode.dst[2:]
-            f.request.scheme = "https" if self.server.config.mode.dst[1] else "http"
+            f.request.scheme = "https" if self.server.config.mode.dst[
+                1] else "http"
 
         f.reply = controller.DummyReply()
         if f.request:
@@ -836,7 +879,7 @@ class FlowMaster(controller.Master):
         try:
             f = file(path, "rb")
             freader = FlowReader(f)
-        except IOError, v:
+        except IOError as v:
             raise FlowReadError(v.strerror)
         return self.load_flows(freader)
 
@@ -877,7 +920,8 @@ class FlowMaster(controller.Master):
             f.backup()
             f.request.is_replay = True
             if f.request.content:
-                f.request.headers["Content-Length"] = [str(len(f.request.content))]
+                f.request.headers[
+                    "Content-Length"] = [str(len(f.request.content))]
             f.response = None
             f.error = None
             self.process_new_request(f)
@@ -1028,7 +1072,7 @@ class FlowReader:
         """
         off = 0
         try:
-            while 1:
+            while True:
                 data = tnetstring.load(self.fo)
                 if tuple(data["version"][:2]) != version.IVERSION[:2]:
                     v = ".".join(str(i) for i in data["version"])
@@ -1037,7 +1081,7 @@ class FlowReader:
                     )
                 off = self.fo.tell()
                 yield handle.protocols[data["type"]]["flow"].from_state(data)
-        except ValueError, v:
+        except ValueError as v:
             # Error is due to EOF
             if self.fo.tell() == off and self.fo.read() == '':
                 return

libmproxy/main.py

@@ -70,27 +70,29 @@ def get_server(dummy_server, options):
     else:
         try:
             return ProxyServer(options)
-        except ProxyServerError, v:
+        except ProxyServerError as v:
             print(str(v), file=sys.stderr)
             sys.exit(1)
 
 
-def mitmproxy():  # pragma: nocover
+def mitmproxy(args=None):  # pragma: nocover
     from . import console
 
     check_versions()
     assert_utf8_env()
 
     parser = cmdline.mitmproxy()
-    options = parser.parse_args()
+    options = parser.parse_args(args)
     if options.quiet:
         options.verbose = 0
 
     proxy_config = process_proxy_options(parser, options)
     console_options = console.Options(**cmdline.get_common_options(options))
     console_options.palette = options.palette
+    console_options.palette_transparent = options.palette_transparent
     console_options.eventlog = options.eventlog
     console_options.intercept = options.intercept
+    console_options.limit = options.limit
 
     server = get_server(console_options.no_server, proxy_config)
 
@@ -101,13 +103,13 @@ def mitmproxy():  # pragma: nocover
         pass
 
 
-def mitmdump():  # pragma: nocover
+def mitmdump(args=None):  # pragma: nocover
     from . import dump
 
     check_versions()
 
     parser = cmdline.mitmdump()
-    options = parser.parse_args()
+    options = parser.parse_args(args)
     if options.quiet:
         options.verbose = 0
         options.flow_detail = 0
@@ -135,13 +137,13 @@ def mitmdump():  # pragma: nocover
         pass
 
 
-def mitmweb():  # pragma: nocover
+def mitmweb(args=None):  # pragma: nocover
     from . import web
 
     check_versions()
     parser = cmdline.mitmweb()
 
-    options = parser.parse_args()
+    options = parser.parse_args(args)
     if options.quiet:
         options.verbose = 0
 

libmproxy/onboarding/app.py

@@ -45,7 +45,10 @@ class PEM(tornado.web.RequestHandler):
     def get(self):
         p = os.path.join(self.request.master.server.config.cadir, self.filename)
         self.set_header("Content-Type", "application/x-x509-ca-cert")
-        self.set_header("Content-Disposition", "inline; filename={}".format(self.filename))
+        self.set_header(
+            "Content-Disposition",
+            "inline; filename={}".format(
+                self.filename))
 
         with open(p, "rb") as f:
             self.write(f.read())
@@ -59,7 +62,10 @@ class P12(tornado.web.RequestHandler):
     def get(self):
         p = os.path.join(self.request.master.server.config.cadir, self.filename)
         self.set_header("Content-Type", "application/x-pkcs12")
-        self.set_header("Content-Disposition", "inline; filename={}".format(self.filename))
+        self.set_header(
+            "Content-Disposition",
+            "inline; filename={}".format(
+                self.filename))
 
         with open(p, "rb") as f:
             self.write(f.read())
@@ -78,7 +84,6 @@ application = tornado.web.Application(
             }
         ),
     ],
-    #debug=True
+    # debug=True
 )
 mapp = Adapter(application)
-

libmproxy/platform/__init__.py

@@ -8,7 +8,7 @@ if sys.platform == "linux2":
 elif sys.platform == "darwin":
     from . import osx
     resolver = osx.Resolver
-elif sys.platform == "freebsd10":
+elif sys.platform.startswith("freebsd"):
     from . import osx
     resolver = osx.Resolver
 elif sys.platform == "win32":

libmproxy/platform/linux.py

@@ -1,4 +1,5 @@
-import socket, struct
+import socket
+import struct
 
 # Python socket module does not have this constant
 SO_ORIGINAL_DST = 80

libmproxy/platform/osx.py

@@ -21,6 +21,7 @@ class Resolver(object):
         peer = csock.getpeername()
         stxt = subprocess.check_output(self.STATECMD, stderr=subprocess.STDOUT)
         if "sudo: a password is required" in stxt:
-            raise RuntimeError("Insufficient privileges to access pfctl. "
+            raise RuntimeError(
+                "Insufficient privileges to access pfctl. "
                 "See http://mitmproxy.org/doc/transparent/osx.html for details.")
         return pf.lookup(peer[0], peer[1], stxt)

libmproxy/platform/pf.py

@@ -13,7 +13,7 @@ def lookup(address, port, s):
         if "ESTABLISHED:ESTABLISHED" in i and spec in i:
             s = i.split()
             if len(s) > 4:
-                if sys.platform == "freebsd10":
+                if sys.platform.startswith("freebsd"):
                     # strip parentheses for FreeBSD pfctl
                     s = s[3][1:-1].split(":")
                 else:

libmproxy/platform/windows.py

@@ -197,9 +197,12 @@ class TransparentProxy(object):
         self.driver = WinDivert()
         self.driver.register()
 
-        self.request_filter = custom_filter or " or ".join(("tcp.DstPort == %d" % p) for p in redirect_ports)
+        self.request_filter = custom_filter or " or ".join(
+            ("tcp.DstPort == %d" %
+             p) for p in redirect_ports)
         self.request_forward_handle = None
-        self.request_forward_thread = threading.Thread(target=self.request_forward)
+        self.request_forward_thread = threading.Thread(
+            target=self.request_forward)
         self.request_forward_thread.daemon = True
 
         self.addr_pid_map = dict()
@@ -235,17 +238,25 @@ class TransparentProxy(object):
         # Block all ICMP requests (which are sent on Windows by default).
         # In layman's terms: If we don't do this, our proxy machine tells the client that it can directly connect to the
         # real gateway if they are on the same network.
-        self.icmp_handle = self.driver.open_handle(filter="icmp", layer=Layer.NETWORK, flags=Flag.DROP)
+        self.icmp_handle = self.driver.open_handle(
+            filter="icmp",
+            layer=Layer.NETWORK,
+            flags=Flag.DROP)
 
-        self.response_handle = self.driver.open_handle(filter=self.response_filter, layer=Layer.NETWORK)
+        self.response_handle = self.driver.open_handle(
+            filter=self.response_filter,
+            layer=Layer.NETWORK)
         self.response_thread.start()
 
         if self.mode == "forward" or self.mode == "both":
-            self.request_forward_handle = self.driver.open_handle(filter=self.request_filter,
+            self.request_forward_handle = self.driver.open_handle(
+                filter=self.request_filter,
                 layer=Layer.NETWORK_FORWARD)
             self.request_forward_thread.start()
         if self.mode == "local" or self.mode == "both":
-            self.request_local_handle = self.driver.open_handle(filter=self.request_filter, layer=Layer.NETWORK)
+            self.request_local_handle = self.driver.open_handle(
+                filter=self.request_filter,
+                layer=Layer.NETWORK)
             self.request_local_thread.start()
 
     def shutdown(self):
@@ -266,14 +277,17 @@ class TransparentProxy(object):
         try:
             raw_packet, metadata = handle.recv()
             return self.driver.parse_packet(raw_packet), metadata
-        except WindowsError, e:
+        except WindowsError as e:
             if e.winerror == 995:
                 return None, None
             else:
                 raise
 
     def fetch_pids(self):
-        ret = windll.iphlpapi.GetTcpTable2(byref(self.tcptable2), byref(self.tcptable2_size), 0)
+        ret = windll.iphlpapi.GetTcpTable2(
+            byref(
+                self.tcptable2), byref(
+                self.tcptable2_size), 0)
         if ret == ERROR_INSUFFICIENT_BUFFER:
             self.tcptable2 = MIB_TCPTABLE2(self.tcptable2_size.value)
             self.fetch_pids()
@@ -299,7 +313,8 @@ class TransparentProxy(object):
                 self.fetch_pids()
 
             # If this fails, we most likely have a connection from an external client to
-            # a local server on 80/443. In this, case we always want to proxy the request.
+            # a local server on 80/443. In this, case we always want to proxy
+            # the request.
             pid = self.addr_pid_map.get(client, None)
 
             if pid not in self.trusted_pids:
@@ -325,7 +340,8 @@ class TransparentProxy(object):
         server = (packet.dst_addr, packet.dst_port)
 
         if client in self.client_server_map:
-            del self.client_server_map[client]  # Force re-add to mark as "newest" entry in the dict.
+            # Force re-add to mark as "newest" entry in the dict.
+            del self.client_server_map[client]
         while len(self.client_server_map) > self.connection_cache_size:
             self.client_server_map.popitem(False)
 
@@ -335,7 +351,8 @@ class TransparentProxy(object):
         metadata.direction = Direction.INBOUND
 
         packet = self.driver.update_packet_checksums(packet)
-        # Use any handle thats on the NETWORK layer - request_local may be unavailable.
+        # Use any handle thats on the NETWORK layer - request_local may be
+        # unavailable.
         self.response_handle.send((packet.raw, metadata))
 
     def response(self):
@@ -361,14 +378,31 @@ class TransparentProxy(object):
 
 
 if __name__ == "__main__":
-    parser = configargparse.ArgumentParser(description="Windows Transparent Proxy")
+    parser = configargparse.ArgumentParser(
-    parser.add_argument('--mode', choices=['forward', 'local', 'both'], default="both",
+        description="Windows Transparent Proxy")
+    parser.add_argument(
+        '--mode',
+        choices=[
+            'forward',
+            'local',
+            'both'],
+        default="both",
         help='redirection operation mode: "forward" to only redirect forwarded packets, '
         '"local" to only redirect packets originating from the local machine')
     group = parser.add_mutually_exclusive_group()
-    group.add_argument("--redirect-ports", nargs="+", type=int, default=[80, 443], metavar="80",
+    group.add_argument(
+        "--redirect-ports",
+        nargs="+",
+        type=int,
+        default=[
+            80,
+            443],
+        metavar="80",
         help="ports that should be forwarded to the proxy")
-    group.add_argument("--custom-filter", default=None, metavar="WINDIVERT_FILTER",
+    group.add_argument(
+        "--custom-filter",
+        default=None,
+        metavar="WINDIVERT_FILTER",
         help="Custom WinDivert interception rule.")
     parser.add_argument("--proxy-addr", default=False,
                         help="Proxy Server Address")

libmproxy/protocol/handle.py

@@ -6,6 +6,7 @@ protocols = {
     'tcp': dict(handler=tcp.TCPHandler)
 }
 
+
 def protocol_handler(protocol):
     """
     @type protocol: str
@@ -14,4 +15,6 @@ def protocol_handler(protocol):
     if protocol in protocols:
         return protocols[protocol]["handler"]
 
-    raise NotImplementedError("Unknown Protocol: %s" % protocol)   # pragma: nocover
+    raise NotImplementedError(
+        "Unknown Protocol: %s" %
+        protocol)   # pragma: nocover

libmproxy/protocol/http.py

@@ -6,15 +6,16 @@ import time
 import copy
 from email.utils import parsedate_tz, formatdate, mktime_tz
 import threading
-from netlib import http, tcp, http_status
+from netlib import http, tcp, http_status, http_cookies
 import netlib.utils
-from netlib.odict import ODict, ODictCaseless
+from netlib import odict
 from .tcp import TCPHandler
 from .primitives import KILL, ProtocolHandler, Flow, Error
 from ..proxy.connection import ServerConnection
 from .. import encoding, utils, controller, stateobject, proxy
 
 HDR_FORM_URLENCODED = "application/x-www-form-urlencoded"
+HDR_FORM_MULTIPART = "multipart/form-data"
 CONTENT_MISSING = 0
 
 
@@ -22,19 +23,6 @@ class KillSignal(Exception):
     pass
 
 
-def get_line(fp):
-    """
-    Get a line, possibly preceded by a blank.
-    """
-    line = fp.readline()
-    if line == "\r\n" or line == "\n":
-        # Possible leftover from previous message
-        line = fp.readline()
-    if line == "":
-        raise tcp.NetLibDisconnect()
-    return line
-
-
 def send_connect_request(conn, host, port, update_state=True):
     upstream_request = HTTPRequest(
         "authority",
@@ -44,7 +32,7 @@ def send_connect_request(conn, host, port, update_state=True):
         port,
         None,
         (1, 1),
-        ODictCaseless(),
+        odict.ODictCaseless(),
         ""
     )
     conn.send(upstream_request.assemble())
@@ -99,7 +87,7 @@ class HTTPMessage(stateobject.StateObject):
                  timestamp_end=None):
         self.httpversion = httpversion
         self.headers = headers
-        """@type: ODictCaseless"""
+        """@type: odict.ODictCaseless"""
         self.content = content
 
         self.timestamp_start = timestamp_start
@@ -107,7 +95,7 @@ class HTTPMessage(stateobject.StateObject):
 
     _stateobject_attributes = dict(
         httpversion=tuple,
-        headers=ODictCaseless,
+        headers=odict.ODictCaseless,
         content=str,
         timestamp_start=float,
         timestamp_end=float
@@ -119,6 +107,8 @@ class HTTPMessage(stateobject.StateObject):
         if short:
             if self.content:
                 ret["contentLength"] = len(self.content)
+            elif self.content == CONTENT_MISSING:
+                ret["contentLength"] = None
             else:
                 ret["contentLength"] = 0
         return ret
@@ -239,7 +229,7 @@ class HTTPRequest(HTTPMessage):
 
         httpversion: HTTP version tuple, e.g. (1,1)
 
-        headers: ODictCaseless object
+        headers: odict.ODictCaseless object
 
         content: Content of the request, None, or CONTENT_MISSING if there
         is content associated, but not present. CONTENT_MISSING evaluates
@@ -277,7 +267,7 @@ class HTTPRequest(HTTPMessage):
         timestamp_end=None,
         form_out=None
     ):
-        assert isinstance(headers, ODictCaseless) or not headers
+        assert isinstance(headers, odict.ODictCaseless) or not headers
         HTTPMessage.__init__(
             self,
             httpversion,
@@ -315,7 +305,18 @@ class HTTPRequest(HTTPMessage):
 
     @classmethod
     def from_state(cls, state):
-        f = cls(None, None, None, None, None, None, None, None, None, None, None)
+        f = cls(
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None)
         f.load_state(state)
         return f
 
@@ -325,78 +326,56 @@ class HTTPRequest(HTTPMessage):
         )
 
     @classmethod
-    def from_stream(cls, rfile, include_body=True, body_size_limit=None):
+    def from_stream(
+            cls,
+            rfile,
+            include_body=True,
+            body_size_limit=None,
+            wfile=None):
         """
         Parse an HTTP request from a file stream
+
+        Args:
+            rfile (file): Input file to read from
+            include_body (bool): Read response body as well
+            body_size_limit (bool): Maximum body size
+            wfile (file): If specified, HTTP Expect headers are handled automatically.
+                          by writing a HTTP 100 CONTINUE response to the stream.
+
+        Returns:
+            HTTPRequest: The HTTP request
+
+        Raises:
+            HttpError: If the input is invalid.
         """
-        httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end = (
+        timestamp_start, timestamp_end = None, None
-            None, None, None, None, None, None, None, None, None, None)
 
         timestamp_start = utils.timestamp()
-
         if hasattr(rfile, "reset_timestamps"):
             rfile.reset_timestamps()
 
-        request_line = get_line(rfile)
+        req = http.read_request(
+            rfile,
+            include_body = include_body,
+            body_size_limit = body_size_limit,
+            wfile = wfile
+        )
 
         if hasattr(rfile, "first_byte_timestamp"):
             # more accurate timestamp_start
             timestamp_start = rfile.first_byte_timestamp
 
-        request_line_parts = http.parse_init(request_line)
-        if not request_line_parts:
-            raise http.HttpError(
-                400,
-                "Bad HTTP request line: %s" % repr(request_line)
-            )
-        method, path, httpversion = request_line_parts
-
-        if path == '*' or path.startswith("/"):
-            form_in = "relative"
-            if not netlib.utils.isascii(path):
-                raise http.HttpError(
-                    400,
-                    "Bad HTTP request line: %s" % repr(request_line)
-                )
-        elif method.upper() == 'CONNECT':
-            form_in = "authority"
-            r = http.parse_init_connect(request_line)
-            if not r:
-                raise http.HttpError(
-                    400,
-                    "Bad HTTP request line: %s" % repr(request_line)
-                )
-            host, port, _ = r
-            path = None
-        else:
-            form_in = "absolute"
-            r = http.parse_init_proxy(request_line)
-            if not r:
-                raise http.HttpError(
-                    400,
-                    "Bad HTTP request line: %s" % repr(request_line)
-                )
-            _, scheme, host, port, path, _ = r
-
-        headers = http.read_headers(rfile)
-        if headers is None:
-            raise http.HttpError(400, "Invalid headers")
-
-        if include_body:
-            content = http.read_http_body(rfile, headers, body_size_limit,
-                                          method, None, True)
         timestamp_end = utils.timestamp()
-
         return HTTPRequest(
-            form_in,
+            req.form_in,
-            method,
+            req.method,
-            scheme,
+            req.scheme,
-            host,
+            req.host,
-            port,
+            req.port,
-            path,
+            req.path,
-            httpversion,
+            req.httpversion,
-            headers,
+            req.headers,
-            content,
+            req.content,
             timestamp_start,
             timestamp_end
         )
@@ -440,11 +419,12 @@ class HTTPRequest(HTTPMessage):
                                               self.host,
                                               self.port)]
 
-        # If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header.
+        # If content is defined (i.e. not None or CONTENT_MISSING), we always
+        # add a content-length header.
         if self.content or self.content == "":
             headers["Content-Length"] = [str(len(self.content))]
 
-        return str(headers)
+        return headers.format()
 
     def _assemble_head(self, form=None):
         return "%s\r\n%s\r\n" % (
@@ -497,9 +477,9 @@ class HTTPRequest(HTTPMessage):
             decode appropriately.
         """
         if self.headers["accept-encoding"]:
-            self.headers["accept-encoding"] = [', '.join(
+            self.headers["accept-encoding"] = [
-                e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0]
+                ', '.join(
-            )]
+                    e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0])]
 
     def update_host_header(self):
         """
@@ -507,15 +487,42 @@ class HTTPRequest(HTTPMessage):
         """
         self.headers["Host"] = [self.host]
 
+    def get_form(self):
+        """
+            Retrieves the URL-encoded or multipart form data, returning an ODict object.
+            Returns an empty ODict if there is no data or the content-type
+            indicates non-form data.
+        """
+        if self.content:
+            if self.headers.in_any("content-type", HDR_FORM_URLENCODED, True):
+                return self.get_form_urlencoded()
+            elif self.headers.in_any("content-type", HDR_FORM_MULTIPART, True):
+                return self.get_form_multipart()
+        return odict.ODict([])
+
     def get_form_urlencoded(self):
         """
             Retrieves the URL-encoded form data, returning an ODict object.
             Returns an empty ODict if there is no data or the content-type
             indicates non-form data.
         """
-        if self.content and self.headers.in_any("content-type", HDR_FORM_URLENCODED, True):
+        if self.content and self.headers.in_any(
-            return ODict(utils.urldecode(self.content))
+                "content-type",
-        return ODict([])
+                HDR_FORM_URLENCODED,
+                True):
+            return odict.ODict(utils.urldecode(self.content))
+        return odict.ODict([])
+
+    def get_form_multipart(self):
+        if self.content and self.headers.in_any(
+                "content-type",
+                HDR_FORM_MULTIPART,
+                True):
+            return odict.ODict(
+                utils.multipartdecode(
+                    self.headers,
+                    self.content))
+        return odict.ODict([])
 
     def set_form_urlencoded(self, odict):
         """
@@ -556,8 +563,8 @@ class HTTPRequest(HTTPMessage):
         """
         _, _, _, _, query, _ = urlparse.urlparse(self.url)
         if query:
-            return ODict(utils.urldecode(query))
+            return odict.ODict(utils.urldecode(query))
-        return ODict([])
+        return odict.ODict([])
 
     def set_query(self, odict):
         """
@@ -588,8 +595,10 @@ class HTTPRequest(HTTPMessage):
             host = self.headers.get_first("host")
         if not host:
             host = self.host
-        host = host.encode("idna")
+        if host:
-        return host
+            return host.encode("idna")
+        else:
+            return None
 
     def pretty_url(self, hostheader):
         if self.form_out == "authority":  # upstream proxy mode
@@ -625,15 +634,22 @@ class HTTPRequest(HTTPMessage):
         self.scheme, self.host, self.port, self.path = parts
 
     def get_cookies(self):
-        cookie_headers = self.headers.get("cookie")
+        """
-        if not cookie_headers:
-            return None
 
-        cookies = []
+            Returns a possibly empty netlib.odict.ODict object.
-        for header in cookie_headers:
+        """
-            pairs = [pair.partition("=") for pair in header.split(';')]
+        ret = odict.ODict()
-            cookies.extend((pair[0], (pair[2], {})) for pair in pairs)
+        for i in self.headers["cookie"]:
-        return dict(cookies)
+            ret.extend(http_cookies.parse_cookie_header(i))
+        return ret
+
+    def set_cookies(self, odict):
+        """
+            Takes an netlib.odict.ODict object. Over-writes any existing Cookie
+            headers.
+        """
+        v = http_cookies.format_cookie_header(odict)
+        self.headers["Cookie"] = [v]
 
     def replace(self, pattern, repl, *args, **kwargs):
         """
@@ -674,9 +690,16 @@ class HTTPResponse(HTTPMessage):
         timestamp_end: Timestamp indicating when request transmission ended
     """
 
-    def __init__(self, httpversion, code, msg, headers, content, timestamp_start=None,
+    def __init__(
+            self,
+            httpversion,
+            code,
+            msg,
+            headers,
+            content,
+            timestamp_start=None,
             timestamp_end=None):
-        assert isinstance(headers, ODictCaseless) or headers is None
+        assert isinstance(headers, odict.ODictCaseless) or headers is None
         HTTPMessage.__init__(
             self,
             httpversion,
@@ -706,7 +729,10 @@ class HTTPResponse(HTTPMessage):
         return f
 
     def __repr__(self):
-        size = utils.pretty_size(len(self.content)) if self.content else "content missing"
+        if self.content:
+            size = netlib.utils.pretty_size(len(self.content))
+        else:
+            size = "content missing"
         return "<HTTPResponse: {code} {msg} ({contenttype}, {size})>".format(
             code=self.code,
             msg=self.msg,
@@ -717,7 +743,12 @@ class HTTPResponse(HTTPMessage):
         )
 
     @classmethod
-    def from_stream(cls, rfile, request_method, include_body=True, body_size_limit=None):
+    def from_stream(
+            cls,
+            rfile,
+            request_method,
+            include_body=True,
+            body_size_limit=None):
         """
         Parse an HTTP response from a file stream
         """
@@ -767,11 +798,12 @@ class HTTPResponse(HTTPMessage):
         if not preserve_transfer_encoding:
             del headers['Transfer-Encoding']
 
-        # If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header.
+        # If content is defined (i.e. not None or CONTENT_MISSING), we always
+        # add a content-length header.
         if self.content or self.content == "":
             headers["Content-Length"] = [str(len(self.content))]
 
-        return str(headers)
+        return headers.format()
 
     def _assemble_head(self, preserve_transfer_encoding=False):
         return '%s\r\n%s\r\n' % (
@@ -850,20 +882,39 @@ class HTTPResponse(HTTPMessage):
             self.headers["set-cookie"] = c
 
     def get_cookies(self):
-        cookie_headers = self.headers.get("set-cookie")
+        """
-        if not cookie_headers:
+            Get the contents of all Set-Cookie headers.
-            return None
 
-        cookies = []
+            Returns a possibly empty ODict, where keys are cookie name strings,
-        for header in cookie_headers:
+            and values are [value, attr] lists. Value is a string, and attr is
-            pairs = [pair.partition("=") for pair in header.split(';')]
+            an ODictCaseless containing cookie attributes. Within attrs, unary
-            cookie_name = pairs[0][0]  # the key of the first key/value pairs
+            attributes (e.g. HTTPOnly) are indicated by a Null value.
-            cookie_value = pairs[0][2]  # the value of the first key/value pairs
+        """
-            cookie_parameters = {
+        ret = []
-                key.strip().lower(): value.strip() for key, sep, value in pairs[1:]
+        for header in self.headers["set-cookie"]:
-            }
+            v = http_cookies.parse_set_cookie_header(header)
-            cookies.append((cookie_name, (cookie_value, cookie_parameters)))
+            if v:
-        return dict(cookies)
+                name, value, attrs = v
+                ret.append([name, [value, attrs]])
+        return odict.ODict(ret)
+
+    def set_cookies(self, odict):
+        """
+            Set the Set-Cookie headers on this response, over-writing existing
+            headers.
+
+            Accepts an ODict of the same format as that returned by get_cookies.
+        """
+        values = []
+        for i in odict.lst:
+            values.append(
+                http_cookies.format_set_cookie_header(
+                    i[0],
+                    i[1][0],
+                    i[1][1]
+                )
+            )
+        self.headers["Set-Cookie"] = values
 
 
 class HTTPFlow(Flow):
@@ -996,7 +1047,7 @@ class HTTPHandler(ProtocolHandler):
                     include_body=False
                 )
                 break
-            except (tcp.NetLibError, http.HttpErrorConnClosed), v:
+            except (tcp.NetLibError, http.HttpErrorConnClosed) as v:
                 self.c.log(
                     "error in server communication: %s" % repr(v),
                     level="debug"
@@ -1041,7 +1092,8 @@ class HTTPHandler(ProtocolHandler):
             try:
                 req = HTTPRequest.from_stream(
                     self.c.client_conn.rfile,
-                    body_size_limit=self.c.config.body_size_limit
+                    body_size_limit=self.c.config.body_size_limit,
+                    wfile=self.c.client_conn.wfile
                 )
             except tcp.NetLibError:
                 # don't throw an error for disconnects that happen
@@ -1066,7 +1118,8 @@ class HTTPHandler(ProtocolHandler):
             if request_reply is None or request_reply == KILL:
                 raise KillSignal()
 
-            self.process_server_address(flow)  # The inline script may have changed request.host
+            # The inline script may have changed request.host
+            self.process_server_address(flow)
 
             if isinstance(request_reply, HTTPResponse):
                 flow.response = request_reply
@@ -1077,7 +1130,9 @@ class HTTPHandler(ProtocolHandler):
             # we can safely set it as the final attribute value here.
             flow.server_conn = self.c.server_conn
 
-            self.c.log("response", "debug", [flow.response._assemble_first_line()])
+            self.c.log(
+                "response", "debug", [
+                    flow.response._assemble_first_line()])
             response_reply = self.c.channel.ask("response", flow)
             if response_reply is None or response_reply == KILL:
                 raise KillSignal()
@@ -1104,7 +1159,8 @@ class HTTPHandler(ProtocolHandler):
                         }
                     )
                 )
-                if not self.process_connect_request((flow.request.host, flow.request.port)):
+                if not self.process_connect_request(
+                        (flow.request.host, flow.request.port)):
                     return False
 
             # If the user has changed the target server on this connection,
@@ -1117,7 +1173,7 @@ class HTTPHandler(ProtocolHandler):
                 http.HttpError,
                 proxy.ProxyError,
                 tcp.NetLibError,
-        ), e:
+        ) as e:
             self.handle_error(e, flow)
         except KillSignal:
             self.c.log("Connection killed", "info")
@@ -1213,7 +1269,8 @@ class HTTPHandler(ProtocolHandler):
         # Determine .scheme, .host and .port attributes
         # For absolute-form requests, they are directly given in the request.
         # For authority-form requests, we only need to determine the request scheme.
-        # For relative-form requests, we need to determine host and port as well.
+        # For relative-form requests, we need to determine host and port as
+        # well.
         if not request.scheme:
             request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http"
         if not request.host:
@@ -1240,7 +1297,8 @@ class HTTPHandler(ProtocolHandler):
                 flow.server_conn = self.c.server_conn
                 self.c.establish_server_connection()
                 self.c.client_conn.send(
-                    'HTTP/1.1 200 Connection established\r\n' +
+                    ('HTTP/%s.%s 200 ' % (request.httpversion[0], request.httpversion[1])) +
+                    'Connection established\r\n' +
                     'Content-Length: 0\r\n' +
                     ('Proxy-agent: %s\r\n' % self.c.config.server_version) +
                     '\r\n'
@@ -1304,7 +1362,8 @@ class HTTPHandler(ProtocolHandler):
             )
 
             if needs_server_change:
-                # force create new connection to the proxy server to reset state
+                # force create new connection to the proxy server to reset
+                # state
                 self.live.change_server(self.c.server_conn.address, force=True)
                 if ssl:
                     send_connect_request(
@@ -1314,8 +1373,9 @@ class HTTPHandler(ProtocolHandler):
                     )
                     self.c.establish_ssl(server=True)
         else:
-            # If we're not in upstream mode, we just want to update the host and
+            # If we're not in upstream mode, we just want to update the host
-            # possibly establish TLS. This is a no op if the addresses match.
+            # and possibly establish TLS. This is a no op if the addresses
+            # match.
             self.live.change_server(address, ssl=ssl)
 
         flow.server_conn = self.c.server_conn
@@ -1323,8 +1383,8 @@ class HTTPHandler(ProtocolHandler):
     def send_response_to_client(self, flow):
         if not flow.response.stream:
             # no streaming:
-            # we already received the full response from the server and can send
+            # we already received the full response from the server and can
-            # it to the client straight away.
+            # send it to the client straight away.
             self.c.client_conn.send(flow.response.assemble())
         else:
             # streaming:
@@ -1356,14 +1416,21 @@ class HTTPHandler(ProtocolHandler):
             semantics. Returns True, if so.
         """
         close_connection = (
-            http.connection_close(flow.request.httpversion, flow.request.headers) or
+            http.connection_close(
-            http.connection_close(flow.response.httpversion, flow.response.headers) or
+                flow.request.httpversion,
-            http.expected_http_body_size(flow.response.headers, False, flow.request.method,
+                flow.request.headers) or http.connection_close(
+                flow.response.httpversion,
+                flow.response.headers) or http.expected_http_body_size(
+                flow.response.headers,
+                False,
+                flow.request.method,
                 flow.response.code) == -1)
         if close_connection:
             if flow.request.form_in == "authority" and flow.response.code == 200:
-                # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313:
+                # Workaround for
-                # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header
+                # https://github.com/mitmproxy/mitmproxy/issues/313: Some
+                # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0
+                # and no Content-Length header
                 pass
             else:
                 return True
@@ -1385,14 +1452,16 @@ class HTTPHandler(ProtocolHandler):
             self.expected_form_out = "relative"
             self.skip_authentication = True
 
-            # In practice, nobody issues a CONNECT request to send unencrypted HTTP requests afterwards.
+            # In practice, nobody issues a CONNECT request to send unencrypted
-            # If we don't delegate to TCP mode, we should always negotiate a SSL connection.
+            # HTTP requests afterwards. If we don't delegate to TCP mode, we
+            # should always negotiate a SSL connection.
             #
-            # FIXME:
+            # FIXME: Turns out the previous statement isn't entirely true.
-            # Turns out the previous statement isn't entirely true. Chrome on Windows CONNECTs to :80
+            # Chrome on Windows CONNECTs to :80 if an explicit proxy is
-            # if an explicit proxy is configured and a websocket connection should be established.
+            # configured and a websocket connection should be established. We
-            # We don't support websocket at the moment, so it fails anyway, but we should come up with
+            # don't support websocket at the moment, so it fails anyway, but we
-            # a better solution to this if we start to support WebSockets.
+            # should come up with a better solution to this if we start to
+            # support WebSockets.
             should_establish_ssl = (
                 address.port in self.c.config.ssl_ports
                 or
@@ -1400,12 +1469,18 @@ class HTTPHandler(ProtocolHandler):
             )
 
             if should_establish_ssl:
-                self.c.log("Received CONNECT request to SSL port. Upgrading to SSL...", "debug")
+                self.c.log(
+                    "Received CONNECT request to SSL port. "
+                    "Upgrading to SSL...", "debug"
+                )
                 self.c.establish_ssl(server=True, client=True)
                 self.c.log("Upgrade to SSL completed.", "debug")
 
             if self.c.config.check_tcp(address):
-                self.c.log("Generic TCP mode for host: %s:%s" % address(), "info")
+                self.c.log(
+                    "Generic TCP mode for host: %s:%s" % address(),
+                    "info"
+                )
                 TCPHandler(self.c).handle_messages()
                 return False
 
@@ -1426,7 +1501,8 @@ class RequestReplayThread(threading.Thread):
 
     def __init__(self, config, flow, masterq, should_exit):
         """
-        masterqueue can be a queue or None, if no scripthooks should be processed.
+            masterqueue can be a queue or None, if no scripthooks should be
+            processed.
         """
         self.config, self.flow = config, flow
         if masterq:
@@ -1452,12 +1528,17 @@ class RequestReplayThread(threading.Thread):
             if not self.flow.response:
                 # In all modes, we directly connect to the server displayed
                 if self.config.mode == "upstream":
-                    server_address = self.config.mode.get_upstream_server(self.flow.client_conn)[2:]
+                    server_address = self.config.mode.get_upstream_server(
+                        self.flow.client_conn
+                    )[2:]
                     server = ServerConnection(server_address)
                     server.connect()
                     if r.scheme == "https":
                         send_connect_request(server, r.host, r.port)
-                        server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni)
+                        server.establish_ssl(
+                            self.config.clientcerts,
+                            sni=self.flow.server_conn.sni
+                        )
                         r.form_out = "relative"
                     else:
                         r.form_out = "absolute"
@@ -1466,12 +1547,18 @@ class RequestReplayThread(threading.Thread):
                     server = ServerConnection(server_address)
                     server.connect()
                     if r.scheme == "https":
-                        server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni)
+                        server.establish_ssl(
+                            self.config.clientcerts,
+                            sni=self.flow.server_conn.sni
+                        )
                     r.form_out = "relative"
                 server.send(r.assemble())
                 self.flow.server_conn = server
-                self.flow.response = HTTPResponse.from_stream(server.rfile, r.method,
+                self.flow.response = HTTPResponse.from_stream(
-                                                              body_size_limit=self.config.body_size_limit)
+                    server.rfile,
+                    r.method,
+                    body_size_limit=self.config.body_size_limit
+                )
             if self.channel:
                 response_reply = self.channel.ask("response", self.flow)
                 if response_reply is None or response_reply == KILL:
@@ -1481,7 +1568,8 @@ class RequestReplayThread(threading.Thread):
             if self.channel:
                 self.channel.ask("error", self.flow)
         except KillSignal:
-            # KillSignal should only be raised if there's a channel in the first place.
+            # KillSignal should only be raised if there's a channel in the
+            # first place.
             self.channel.tell("log", proxy.Log("Connection killed", "info"))
         finally:
             r.form_out = form_out_backup

libmproxy/protocol/primitives.py

@@ -24,6 +24,7 @@ class Error(stateobject.StateObject):
             msg: Message describing the error
             timestamp: Seconds since the epoch
     """
+
     def __init__(self, msg, timestamp=None):
         """
         @type msg: str
@@ -59,6 +60,7 @@ class Flow(stateobject.StateObject):
     A Flow is a collection of objects representing a single transaction.
     This class is usually subclassed for each protocol, e.g. HTTPFlow.
     """
+
     def __init__(self, type, client_conn, server_conn, live=None):
         self.type = type
         self.id = str(uuid.uuid4())
@@ -165,12 +167,12 @@ class Flow(stateobject.StateObject):
         master.handle_accept_intercept(self)
 
 
-
 class ProtocolHandler(object):
     """
     A ProtocolHandler implements an application-layer protocol, e.g. HTTP.
     See: libmproxy.protocol.http.HTTPHandler
     """
+
     def __init__(self, c):
         self.c = c
         """@type: libmproxy.proxy.server.ConnectionHandler"""
@@ -209,13 +211,20 @@ class LiveConnection(object):
     interface with a live connection, without exposing the internals
     of the ConnectionHandler.
     """
+
     def __init__(self, c):
         self.c = c
         """@type: libmproxy.proxy.server.ConnectionHandler"""
         self._backup_server_conn = None
         """@type: libmproxy.proxy.connection.ServerConnection"""
 
-    def change_server(self, address, ssl=None, sni=None, force=False, persistent_change=False):
+    def change_server(
+            self,
+            address,
+            ssl=None,
+            sni=None,
+            force=False,
+            persistent_change=False):
         """
         Change the server connection to the specified address.
         @returns:

libmproxy/protocol/tcp.py

@@ -79,7 +79,8 @@ class TCPHandler(ProtocolHandler):
                                 ),
                                 "info"
                             )
-                        # Do not use dst.connection.send here, which may raise OpenSSL-specific errors.
+                        # Do not use dst.connection.send here, which may raise
+                        # OpenSSL-specific errors.
                         dst.send(contents)
                     else:
                         # socket.socket.send supports raw bytearrays/memoryviews

libmproxy/proxy/config.py

@@ -81,16 +81,27 @@ class ProxyConfig:
         self.check_tcp = HostMatcher(tcp_hosts)
         self.authenticator = authenticator
         self.cadir = os.path.expanduser(cadir)
-        self.certstore = certutils.CertStore.from_store(self.cadir, CONF_BASENAME)
+        self.certstore = certutils.CertStore.from_store(
+            self.cadir,
+            CONF_BASENAME)
         for spec, cert in certs:
             self.certstore.add_cert_file(spec, cert)
         self.certforward = certforward
-        self.openssl_method_client, self.openssl_options_client = version_to_openssl(ssl_version_client)
+        self.openssl_method_client, self.openssl_options_client = version_to_openssl(
-        self.openssl_method_server, self.openssl_options_server = version_to_openssl(ssl_version_server)
+            ssl_version_client)
+        self.openssl_method_server, self.openssl_options_server = version_to_openssl(
+            ssl_version_server)
         self.ssl_ports = ssl_ports
 
 
-sslversion_choices = ("all", "secure", "SSLv2", "SSLv3", "TLSv1", "TLSv1_1", "TLSv1_2")
+sslversion_choices = (
+    "all",
+    "secure",
+    "SSLv2",
+    "SSLv3",
+    "TLSv1",
+    "TLSv1_1",
+    "TLSv1_2")
 
 
 def version_to_openssl(version):
@@ -119,7 +130,8 @@ def process_proxy_options(parser, options):
     if options.transparent_proxy:
         c += 1
         if not platform.resolver:
-            return parser.error("Transparent mode not supported on this platform.")
+            return parser.error(
+                "Transparent mode not supported on this platform.")
         mode = "transparent"
     if options.socks_proxy:
         c += 1
@@ -133,28 +145,33 @@ def process_proxy_options(parser, options):
         mode = "upstream"
         upstream_server = options.upstream_proxy
     if c > 1:
-        return parser.error("Transparent, SOCKS5, reverse and upstream proxy mode "
+        return parser.error(
+            "Transparent, SOCKS5, reverse and upstream proxy mode "
             "are mutually exclusive.")
 
     if options.clientcerts:
         options.clientcerts = os.path.expanduser(options.clientcerts)
-        if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts):
+        if not os.path.exists(
+                options.clientcerts) or not os.path.isdir(
+                options.clientcerts):
             return parser.error(
-                "Client certificate directory does not exist or is not a directory: %s" % options.clientcerts
+                "Client certificate directory does not exist or is not a directory: %s" %
-            )
+                options.clientcerts)
 
     if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd):
         if options.auth_singleuser:
             if len(options.auth_singleuser.split(':')) != 2:
-                return parser.error("Invalid single-user specification. Please use the format username:password")
+                return parser.error(
+                    "Invalid single-user specification. Please use the format username:password")
             username, password = options.auth_singleuser.split(':')
             password_manager = http_auth.PassManSingleUser(username, password)
         elif options.auth_nonanonymous:
             password_manager = http_auth.PassManNonAnon()
         elif options.auth_htpasswd:
             try:
-                password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd)
+                password_manager = http_auth.PassManHtpasswd(
-            except ValueError, v:
+                    options.auth_htpasswd)
+            except ValueError as v:
                 return parser.error(v.message)
         authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")
     else:
@@ -203,15 +220,18 @@ def process_proxy_options(parser, options):
 def ssl_option_group(parser):
     group = parser.add_argument_group("SSL")
     group.add_argument(
-        "--cert", dest='certs', default=[], type=str,
+        "--cert",
-        metavar="SPEC", action="append",
+        dest='certs',
+        default=[],
+        type=str,
+        metavar="SPEC",
+        action="append",
         help='Add an SSL certificate. SPEC is of the form "[domain=]path". '
         'The domain may include a wildcard, and is equal to "*" if not specified. '
         'The file at path is a certificate in PEM format. If a private key is included in the PEM, '
         'it is used, else the default key in the conf dir is used. '
         'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. '
-             'Can be passed multiple times.'
+        'Can be passed multiple times.')
-    )
     group.add_argument(
         "--cert-forward", action="store_true",
         dest="certforward", default=False,
@@ -238,11 +258,15 @@ def ssl_option_group(parser):
         help="Don't connect to upstream server to look up certificate details."
     )
     group.add_argument(
-        "--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS),
+        "--ssl-port",
+        action="append",
+        type=int,
+        dest="ssl_ports",
+        default=list(TRANSPARENT_SSL_PORTS),
         metavar="PORT",
         help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. "
-             "Defaults to %s." % str(TRANSPARENT_SSL_PORTS)
+        "Defaults to %s." %
-    )
+        str(TRANSPARENT_SSL_PORTS))
     group.add_argument(
         "--ssl-version-client", dest="ssl_version_client",
         default="secure", action="store",

libmproxy/proxy/connection.py

@@ -7,7 +7,9 @@ from .. import stateobject, utils
 
 class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
     def __init__(self, client_connection, address, server):
-        if client_connection:  # Eventually, this object is restored from state. We don't have a connection then.
+        # Eventually, this object is restored from state. We don't have a
+        # connection then.
+        if client_connection:
             tcp.BaseHandler.__init__(self, client_connection, address, server)
         else:
             self.connection = None
@@ -39,15 +41,18 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
     def get_state(self, short=False):
         d = super(ClientConnection, self).get_state(short)
         d.update(
-            address={"address": self.address(), "use_ipv6": self.address.use_ipv6},
+            address={
-            clientcert=self.cert.to_pem() if self.clientcert else None
+                "address": self.address(),
-        )
+                "use_ipv6": self.address.use_ipv6},
+            clientcert=self.cert.to_pem() if self.clientcert else None)
         return d
 
     def load_state(self, state):
         super(ClientConnection, self).load_state(state)
-        self.address = tcp.Address(**state["address"]) if state["address"] else None
+        self.address = tcp.Address(
-        self.clientcert = certutils.SSLCert.from_pem(state["clientcert"]) if state["clientcert"] else None
+            **state["address"]) if state["address"] else None
+        self.clientcert = certutils.SSLCert.from_pem(
+            state["clientcert"]) if state["clientcert"] else None
 
     def copy(self):
         return copy.copy(self)
@@ -122,9 +127,12 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
     def load_state(self, state):
         super(ServerConnection, self).load_state(state)
 
-        self.address = tcp.Address(**state["address"]) if state["address"] else None
+        self.address = tcp.Address(
-        self.source_address = tcp.Address(**state["source_address"]) if state["source_address"] else None
+            **state["address"]) if state["address"] else None
-        self.cert = certutils.SSLCert.from_pem(state["cert"]) if state["cert"] else None
+        self.source_address = tcp.Address(
+            **state["source_address"]) if state["source_address"] else None
+        self.cert = certutils.SSLCert.from_pem(
+            state["cert"]) if state["cert"] else None
 
     @classmethod
     def from_state(cls, state):
@@ -147,7 +155,9 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
     def establish_ssl(self, clientcerts, sni, **kwargs):
         clientcert = None
         if clientcerts:
-            path = os.path.join(clientcerts, self.address.host.encode("idna")) + ".pem"
+            path = os.path.join(
+                clientcerts,
+                self.address.host.encode("idna")) + ".pem"
             if os.path.exists(path):
                 clientcert = path
         self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs)

libmproxy/proxy/primitives.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 from netlib import socks
 
+
 class ProxyError(Exception):
     def __init__(self, code, message, headers=None):
         super(ProxyError, self).__init__(message)
@@ -61,7 +62,7 @@ class TransparentProxyMode(ProxyMode):
     def get_upstream_server(self, client_conn):
         try:
             dst = self.resolver.original_addr(client_conn.connection)
-        except Exception, e:
+        except Exception as e:
             raise ProxyError(502, "Transparent mode failure: %s" % str(e))
 
         if dst[1] in self.sslports:
@@ -87,7 +88,9 @@ class Socks5ProxyMode(ProxyMode):
                 guess = ""
             raise socks.SocksError(
                 socks.REP.GENERAL_SOCKS_SERVER_FAILURE,
-                guess + "Invalid SOCKS version. Expected 0x05, got 0x%x" % msg.ver)
+                guess +
+                "Invalid SOCKS version. Expected 0x05, got 0x%x" %
+                msg.ver)
 
     def get_upstream_server(self, client_conn):
         try:
@@ -117,13 +120,15 @@ class Socks5ProxyMode(ProxyMode):
                     "mitmproxy only supports SOCKS5 CONNECT."
                 )
 
-            # We do not connect here yet, as the clientconnect event has not been handled yet.
+            # We do not connect here yet, as the clientconnect event has not
+            # been handled yet.
 
             connect_reply = socks.Message(
                 socks.VERSION.SOCKS5,
                 socks.REP.SUCCEEDED,
                 socks.ATYP.DOMAINNAME,
-                client_conn.address  # dummy value, we don't have an upstream connection yet.
+                # dummy value, we don't have an upstream connection yet.
+                client_conn.address
             )
             connect_reply.to_file(client_conn.wfile)
             client_conn.wfile.flush()