Merge remote-tracking branch 'upstream/master' into print-bracket-fix

Conflicts:
	examples/har_extractor.py
	examples/nonblocking.py
	examples/read_dumpfile
	libmproxy/web/app.py

.env

@@ -0,0 +1,5 @@
+DIR="${0%/*}"
+if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then
+    echo "Activating mitmproxy virtualenv..."
+    source "$DIR/../venv.mitmproxy/bin/activate"
+fi

CHANGELOG

@@ -1,3 +1,43 @@
+18 May 2015: mitmproxy 0.12
+
+    * mitmproxy console: Significant revamp of the UI. The major changes are
+      listed below, and in addition almost every aspect of the UI has
+      been tweaked, and performance has improved significantly.
+
+    * mitmproxy console: A new options screen has been created ("o" shortcut),
+      and many options that were previously manipulated directly via a
+      keybinding have been moved there.
+
+    * mitmproxy console: Big improvement in palettes. This includes improvements
+      to all colour schemes. Palettes now set the terminal background colour by
+      default, and a new --palette-transparent option has been added to disable
+      this.
+
+    * mitmproxy console: g/G shortcuts throughout mitmproxy console to jump
+      to the beginning/end of the current view.
+
+    * mitmproxy console: switch  palettes on the fly from the options screen.
+
+    * mitmproxy console: A cookie editor has been added for mitmproxy console
+      at long last.
+
+    * mitmproxy console: Various components of requests and responses can be
+      copied to the clipboard from mitmproxy - thanks to @marceloglezer.
+
+    * Support for creating new requests from scratch in mitmproxy console (@marceloglezer).
+
+    * SSLKEYLOGFILE environment variable to specify a logging location for TLS
+      master keys. This can be used with tools like Wireshark to allow TLS
+      decoding.
+
+    * Server facing SSL cipher suite specification (thanks to Jim Shaver).
+
+    * Official support for transparent proxying on FreeBSD - thanks to Mike C
+      (http://github.com/mike-pt).
+
+    * Many other small bugfixes and improvemenets throughout the project.
+
+
 29 Dec 2014: mitmproxy 0.11.2:
 
     * Configuration files - mitmproxy.conf, mitmdump.conf, common.conf in the
@@ -8,11 +48,11 @@
 
 15 November 2014: mitmproxy 0.11.1:
 
-    * Bug fixes: connection leaks some crashes 
+    * Bug fixes: connection leaks some crashes
 
 
 7 November 2014: mitmproxy 0.11:
-    
+
     * Performance improvements for mitmproxy console
 
     * SOCKS5 proxy mode allows mitmproxy to act as a SOCKS5 proxy server
@@ -26,7 +66,7 @@
     * Finer-grained control of traffic replay, including options to ignore
       contents or parameters when matching flows (marcelo.glezer@gmail.com)
 
-    * Pass arguments to inline scripts 
+    * Pass arguments to inline scripts
 
     * Configurable size limit on HTTP request and response bodies
 
@@ -56,7 +96,7 @@
 
 
 28 January 2014: mitmproxy 0.10:
-	
+
     * Support for multiple scripts and multiple script arguments
 
     * Easy certificate install through the in-proxy web app, which is now
@@ -76,7 +116,7 @@
 25 August 2013: mitmproxy 0.9.2:
 
     * Improvements to the mitmproxywrapper.py helper script for OSX.
-    
+
     * Don't take minor version into account when checking for serialized file
       compatibility.
 
@@ -90,38 +130,38 @@
       valid IDNA-encoded names.
 
     * Display transfer rates for responses in the flow list.
-    
+
     * Many other small bugfixes and improvements.
-    
+
 
 
 
 16 June 2013: mitmproxy 0.9.1:
 
     * Use "correct" case for Content-Type headers added by mitmproxy.
-    
+
-    * Make UTF environment detection more robust. 
+    * Make UTF environment detection more robust.
 
     * Improved MIME-type detection for viewers.
-    
+
     * Always read files in binary mode (Windows compatibility fix).
-    
+
     * Some developer documentation.
-    
+
 
 15 May 2013: mitmproxy 0.9:
 
     * Upstream certs mode is now the default.
 
     * Add a WSGI container that lets you host in-proxy web applications.
-    
+
     * Full transparent proxy support for Linux and OSX.
-    
+
     * Introduce netlib, a common codebase for mitmproxy and pathod
       (http://github.com/cortesi/netlib).
 
     * Full support for SNI.
-    
+
     * Color palettes for mitmproxy, tailored for light and dark terminal
       backgrounds.
 
@@ -132,12 +172,12 @@
       match asset flows (js, images, css).
 
     * Follow mode in mitmproxy ("F" shortcut) to "tail" flows as they arrive.
-    
+
     * --dummy-certs option to specify and preserve the dummy certificate
       directory.
 
     * Server replay from the current captured buffer.
-    
+
     * Huge improvements in content views. We now have viewers for AMF, HTML,
       JSON, Javascript, images, XML, URL-encoded forms, as well as hexadecimal
       and raw views.
@@ -146,7 +186,7 @@
       on flows, based on a matching pattern.
 
     * A graphical editor for path components in mitmproxy.
-    
+
     * A small set of standard user-agent strings, which can be used easily in
       the header editor.
 

CONTRIBUTORS

@@ -1,44 +1,59 @@
-   902	Aldo Cortesi
+  1067	Aldo Cortesi
-   323	Maximilian Hils
+   542	Maximilian Hils
+    76	Marcelo Glezer
     18	Henrik Nordstrom
     13	Thomas Roth
     12	Pedro Worcel
     11	Stephen Altamirano
+    11	Justus Wingert
+    11	Jim Shaver
     10	András Veres-Szentkirályi
+     9	Legend Tang
      8	Rouli
      8	Jason A. Novak
      7	Alexis Hildebrandt
-     5	Tomaz Muraus
-     5	Brad Peabody
      5	Matthias Urlichs
+     5	Brad Peabody
+     5	Tomaz Muraus
+     5	elitest
      4	root
-     4	Marc Liyanage
      4	Valtteri Virtanen
+     4	Wade 524
      4	Bryan Bishop
+     4	Youhei Sakurai
+     4	Marc Liyanage
      3	Chris Neasbitt
      3	Zack B
-     3	Eli Shvartsman
      3	Kyle Manna
-     2	Michael Frister
+     3	Eli Shvartsman
-     2	Bennett Blodinger
+     2	Choongwoo Han
-     2	Jim Lloyd
      2	Rob Wills
      2	israel
-     2	Jaime Soriano Pastor
-     2	Heikki Hannikainen
      2	Mark E. Haase
+     2	Jaime Soriano Pastor
+     2	Jim Lloyd
+     2	Heikki Hannikainen
+     2	Krzysztof Bielicki
+     2	Bennett Blodinger
+     2	Michael Frister
      2	alts
+     1	Yuangxuan Wang
+     1	capt8bit
      1	davidpshaw
      1	deployable
      1	joebowbeer
      1	meeee
+     1	michaeljau
+     1	peralta
      1	phil plante
-     1	Michael Bisbjerg
+     1	sentient07
+     1	vzvu3k6k
      1	Andy Smith
      1	Dan Wilbraham
      1	David Shaw
      1	Eric Entzel
      1	Felix Wolfsteller
+     1	Gabriel Kirkpatrick
      1	Henrik Nordström
      1	Ivaylo Popov
      1	JC
@@ -46,20 +61,28 @@
      1	James Billingham
      1	Jean Regisser
      1	Kit Randel
-     1	Marcelo Glezer
+     1	Lucas Cimon
      1	Mathieu Mitchell
+     1	Michael Bisbjerg
+     1	Mike C
      1	Mikhail Korobov
+     1	Nick HS
+     1	Nick Raptis
      1	Nicolas Esteves
      1	Oleksandr Sheremet
      1	Paul
      1	Rich Somerfield
      1	Rory McCann
      1	Rune Halvorsen
+     1	Ryo Onodera
      1	Sahn Lam
      1	Seppo Yli-Olli
      1	Sergey Chipiga
+     1	Steve Phillips
      1	Steven Van Acker
+     1	Suyash
+     1	Tarashish Mishra
+     1	Terry Long
      1	Ulrich Petri
      1	Vyacheslav Bakhmutov
-     1	Yuangxuan Wang
+     1	Wade Catron
-     1	capt8bit

README.mkd

@@ -75,6 +75,10 @@ command:
 
 ```$ mitmdump --version```
 
+For convenience, the project includes an
+[autoenv](https://github.com/kennethreitz/autoenv) file
+([.env](https://github.com/mitmproxy/mitmproxy/blob/master/.env)) that
+auto-activates the virtualenv when you cd into the mitmproxy directory.
 
 
 ### Testing

check_coding_style.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+autopep8 -i -r -a -a .
+if [[ -n "$(git status -s)" ]]; then
+  echo "autopep8 yielded the following changes:"
+  git status -s
+  git --no-pager diff
+  exit 1
+fi
+
+autoflake -i -r --remove-all-unused-imports --remove-unused-variables .
+if [[ -n "$(git status -s)" ]]; then
+  echo "autoflake yielded the following changes:"
+  git status -s
+  git --no-pager diff
+  exit 1
+fi
+
+echo "Coding style seems to be ok."
+exit 0

dev

@@ -1,11 +1,12 @@
-#!/bin/sh
+#!/bin/bash
+set -e
 VENV=../venv.mitmproxy
 
-virtualenv $VENV
+python -m virtualenv $VENV
 source $VENV/bin/activate
 pip install --src .. -r requirements.txt
 
 echo ""
 echo "* Created virtualenv environment in $VENV."
 echo "* Installed all dependencies into the virtualenv."
-echo "* Activated virtualenv environment."
+echo "* Activated virtualenv environment."

dev.bat

@@ -2,8 +2,11 @@
 set VENV=..\venv.mitmproxy
 
 virtualenv %VENV%
+if %errorlevel% neq 0 exit /b %errorlevel%
 call %VENV%\Scripts\activate.bat
+if %errorlevel% neq 0 exit /b %errorlevel%
 pip install --src .. -r requirements.txt
+if %errorlevel% neq 0 exit /b %errorlevel%
 
 echo.
 echo * Created virtualenv environment in %VENV%.

doc-src/02-app.css

@@ -0,0 +1,43 @@
+.masthead {
+  text-align: center;
+  border-bottom: 0;
+}
+.frontpage .talks div {
+  margin-bottom: 10px;
+}
+.nav-sidebar {
+  background-color: #f0f0f0;
+  margin-bottom: 20px;
+}
+.nav-sidebar li {
+  line-height: 1.1;
+}
+.nav-sidebar li > a,
+.nav-sidebar .nav-header {
+  padding-left: 20px;
+}
+.nav-sidebar .nav-header {
+  margin-top: 1em;
+  font-size: 1.2em;
+  font-weight: bold;
+}
+.nav-sidebar .active > a,
+.nav-sidebar .active > a:hover,
+.nav-sidebar .active > a:focus {
+  color: #fff;
+  background-color: #428bca;
+}
+.tablenum {
+  font-weight: bold;
+}
+.nowrap {
+  white-space: nowrap;
+}
+.page-header {
+  margin: 0px 0 22px;
+}
+.page-header h1 {
+  margin-top: 0px;
+}
+
+/*# sourceMappingURL=02-app.css.map */

doc-src/02-docstyle.css

@@ -1,20 +0,0 @@
-body {
-    padding-top: 60px;
-    padding-bottom: 40px;
-}
-
-.tablenum {
-    font-weight: bold;
-} 
-
-.nowrap {
-     white-space: nowrap;
-}
-
-h1 {
-    line-height: 1.1;
-}
-
-.page-header {
-    margin: 0px 0 22px;
-}

doc-src/_layout.html

@@ -1,36 +1,44 @@
-<div class="navbar navbar-fixed-top">
+<!DOCTYPE html>
-  <div class="navbar-inner">
+<html>
-    <div class="container">
+    <head>
-      <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+        <title>@!pageTitle!@</title>
-        <span class="icon-bar"></span>
+        $!header!$
-        <span class="icon-bar"></span>
+    </head>
-        <span class="icon-bar"></span>
+    <body>
-      </a>
+        <div class="navbar navbar-default navbar-static-top">
-      <a class="brand" href="@!urlTo(idxpath)!@">mitmproxy $!VERSION!$ docs</a>
+            <div class="container">
-      </div><!--/.nav-collapse -->
+                <div class="navbar-header">
-    </div>
+                    <a class="navbar-brand" href="@!urlTo("/index.html")!@">
-  </div>
+                        <img height="20px" src="@!urlTo("mitmproxy-long.png")!@"/>
-</div>
+                    </a>
-
+                </div>
-<div class="container">
+                <div class="navbar-header navbar-right">
-  <div class="row">
+                    <a class="navbar-brand" hre="#">$!VERSION!$ docs</a>
-    <div class="span3">
+                </div>
-      <div class="well sidebar-nav">
+            </div>
-            $!navbar!$
-      </div>
-    </div>
-
-    <div class="span9">
-        <div class="page-header">
-        <h1>@!this.title!@</h1>
         </div>
-        $!body!$
-    </div>
-  </div>
 
-  <hr>
 
-  <footer>
+        <div class="container">
-    <p>@!copyright!@</p>
+          <div class="row">
-  </footer>
+            <div class="col-md-3">
-</div>
+                $!navbar!$
+            </div>
+            <div class="col-md-9">
+                <div class="page-header">
+                <h1>@!this.title!@</h1>
+                </div>
+                $!body!$
+            </div>
+          </div>
+        </div>
+
+        <div class="container">
+          <hr>
+
+          <footer>
+            <p>@!copyright!@</p>
+          </footer>
+        </div>
+    </body>
+</html>

doc-src/_nav.html

@@ -1,6 +1,7 @@
-<ul class="nav nav-list">
+<ul class="nav nav-sidebar">
     $!nav(idxpath, this, state)!$
     $!nav("install.html", this, state)!$
+    $!nav("certinstall.html", this, state)!$
     $!nav("howmitmproxy.html", this, state)!$
     $!nav("modes.html", this, state)!$
 
@@ -28,23 +29,12 @@
         $!nav("upstreamcerts.html", this, state)!$
 
 
-    <li class="nav-header">Installing Certificates</li>
-        $!nav("ssl.html", this, state)!$
-        $!nav("certinstall/webapp.html", this, state)!$
-        $!nav("certinstall/android.html", this, state)!$
-        $!nav("certinstall/firefox.html", this, state)!$
-        $!nav("certinstall/ios.html", this, state)!$
-        $!nav("certinstall/ios-simulator.html", this, state)!$
-        $!nav("certinstall/java.html", this, state)!$
-        $!nav("certinstall/osx.html", this, state)!$
-        $!nav("certinstall/windows7.html", this, state)!$
-
     <li class="nav-header">Transparent Proxying</li>
         $!nav("transparent.html", this, state)!$
         $!nav("transparent/linux.html", this, state)!$
         $!nav("transparent/osx.html", this, state)!$
 
-    <li class="nav-header">Scripting mitmproxy</li>
+    <li class="nav-header">Scripting</li>
         $!nav("scripting/inlinescripts.html", this, state)!$
         $!nav("scripting/libmproxy.html", this, state)!$
 

doc-src/_websitelayout.html

@@ -1,42 +0,0 @@
-<div class="navbar navbar-fixed-top">
-  <div class="navbar-inner">
-    <div class="container">
-      <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-        <span class="icon-bar"></span>
-      </a>
-      <a class="brand" href="@!urlTo(idxpath)!@">mitmproxy</a>
-      <div class="nav">
-        <ul class="nav">
-          <li $!'class="active"' if this.match("/index.html", True) else ""!$> <a href="@!top!@/index.html">home</a> </li>
-          <li $!'class="active"' if this.under("/doc") else ""!$><a href="@!top!@/doc/index.html">docs</a></li>
-          <li $!'class="active"' if this.under("/about.html") else ""!$><a href="@!top!@/about.html">about</a></li>
-        </ul>
-      </div>
-    </div>
-  </div>
-</div>
-
-<div class="container">
-  <div class="row">
-
-    <div class="span3">
-      <div class="well sidebar-nav">
-            $!navbar!$
-      </div>
-    </div>
-    <div class="span9">
-        <div class="page-header">
-        <h1>@!this.title!@</h1>
-        </div>
-        $!body!$
-    </div>
-  </div>
-
-  <hr>
-
-  <footer>
-    <p>@!copyright!@</p>
-  </footer>
-</div>

doc-src/certinstall.html

@@ -0,0 +1,151 @@
+## On This Page
+
+* [Introduction](#docIntro)
+* [Quick Setup](#docQuick)
+* [Installing the mitmproxy CA certificate manually](#docManual)
+* [More on mitmproxy certificates](#docMore)
+* [CA and cert files](#docCertfiles)
+* [Using a custom certificate](#docCustom)
+* [Using a client side certificate](#docClient)
+* [Using a custom certificate authority](#docCA)
+
+## <a id="docIntro"></a>Introduction
+
+Mitmproxy can decrypt encrypted traffic on the fly, as long as the client
+trusts its built-in certificate authority. Usually this means that the
+mitmproxy CA certificates have to be installed on the client device.
+
+## <a id="docQuick"></a>Quick Setup
+
+By far the easiest way to install the mitmproxy certificates is to use the
+built-in certificate installation app. To do this, just start mitmproxy and
+configure your target device with the correct proxy settings. Now start a
+browser on the device, and visit the magic domain **mitm.it**. You should see
+something like this:
+
+<img src="@!urlTo("certinstall-webapp.png")!@"></img>
+
+Click on the relevant icon, and follow the setup instructions for the platform
+you're on, and you are good to go.
+
+
+## <a id="docManual"></a>Installing the mitmproxy CA certificate manually
+
+Sometimes using the quick install app is not an option - Java or the IOS
+similator spring to mind - or you just need to do it manually for some other
+reason. Below is a list of pointers to manual certificate installation
+documentation for some common platforms:
+
+<table class="table">
+    <tr>
+        <td><a href="https://github.com/ADVTOOLS/ADVTrustStore#how-to-use-advtruststore"</a>iOS Simulator</td>
+        <td><a href="http://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html">Java</a></td>
+    </tr>
+    <tr>
+        <td><a href="http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377">iOS</a></td>
+        <td><a href="http://wiki.cacert.org/FAQ/ImportRootCert#Android_Phones_.26_Tablets">Android/Android Simulator</a></td>
+    </tr>
+
+    <tr>
+        <td><a href="http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7">Windows</a></td>
+        <td><a href="https://support.apple.com/kb/PH7297?locale=en_US">Mac OS X</a></td>
+    </tr>
+    <tr>
+        <td><a href="http://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861">Ubuntu/Debian</a></td>
+        <td><a href="https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox">Firefox</a></td>
+    </tr>
+    <tr>
+        <td><a href="https://code.google.com/p/chromium/wiki/LinuxCertManagement">Chrome on Linux</a></td>
+    </tr>
+
+</table>
+
+## <a id="docMore"></a>More on mitmproxy certificates
+
+The first time __mitmproxy__ or __mitmdump__ is run, the mitmproxy Certificate
+Authority(CA) is created in the config directory (~/.mitmproxy by default).
+This CA is used for on-the-fly generation of dummy certificates for each of the
+SSL sites that your client visits. Since your browser won't trust the
+__mitmproxy__ CA out of the box , you will see an SSL certificate warning every
+time you visit a new SSL domain through __mitmproxy__. When you are testing a
+single site through a browser, just accepting the bogus SSL cert manually is
+not too much trouble, but there are a many circumstances where you will want to
+configure your testing system or browser to trust the __mitmproxy__ CA as a
+signing root authority.
+
+
+## <a id="docCertfiles"></a>CA and cert files
+
+The files created by mitmproxy in the .mitmproxy directory are as follows:
+
+<table class="table">
+    <tr>
+        <td class="nowrap">mitmproxy-ca.pem</td>
+        <td>The private key and certificate in PEM format.</td>
+    </tr>
+    <tr>
+        <td class="nowrap">mitmproxy-ca-cert.pem</td>
+        <td>The certificate in PEM format. Use this to distribute to most
+        non-Windows platforms.</td>
+    </tr>
+    <tr>
+        <td class="nowrap">mitmproxy-ca-cert.p12</td>
+        <td>The certificate in PKCS12 format. For use on Windows.</td>
+    </tr>
+    <tr>
+        <td class="nowrap">mitmproxy-ca-cert.cer</td>
+        <td>Same file as .pem, but with an extension expected by some Android
+        devices.</td>
+    </tr>
+</table>
+
+
+## <a id="docCustom"></a>Using a custom certificate
+
+You can use your own certificate by passing the <kbd>--cert</kbd> option to
+mitmproxy. mitmproxy then uses the provided certificate for interception of the
+specified domains instead of generating a certificate signed by its own CA.
+
+The certificate file is expected to be in the PEM format.  You can include
+intermediary certificates right below your leaf certificate, so that you PEM
+file roughly looks like this:
+
+<pre>
+-----BEGIN PRIVATE KEY-----
+&lt;private key&gt;
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+&lt;cert&gt;
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+&lt;intermediary cert (optional)&gt;
+-----END CERTIFICATE-----
+</pre>
+
+For example, you can generate a certificate in this format using these instructions:
+
+<pre class="terminal">
+$ openssl genrsa -out cert.key 2048
+$ openssl req -new -x509 -key cert.key -out cert.crt
+    (Specify the mitm domain as Common Name, e.g. *.google.com)
+$ cat cert.key cert.crt > cert.pem
+$ mitmproxy --cert=cert.pem
+</pre>
+
+## <a id="docClient"></a>Using a client side certificate
+
+You can use a client certificate by passing the <kbd>--client-certs
+DIRECTORY</kbd> option to mitmproxy.  If you visit example.org, mitmproxy looks
+for a file named example.org.pem in the specified directory and uses this as
+the client cert. The certificate file needs to be in the PEM format and should
+contain both the unencrypted private key and the certificate.
+
+
+## <a id="docCA"></a>Using a custom certificate authority
+
+By default, mitmproxy will use <samp>~/.mitmproxy/mitmproxy-ca.pem</samp> as
+the certificate authority to generate certificates for all domains for which no
+custom certificate is provided (see above). You can use your own certificate
+authority by passing the <kbd>--confdir</kbd> option to mitmproxy. Mitmproxy
+will then look for <samp>mitmproxy-ca.pem</samp> in the specified directory. If
+no such file exists, it will be generated automatically.

doc-src/certinstall/android.html

@@ -1,53 +0,0 @@
-The proxy situation on Android is [an
-embarrasment](http://code.google.com/p/android/issues/detail?id=1273).  It's
-scarcely credible, but Android didn't have a global proxy setting at all until
-quite recently, and it's still not supported on many common Android versions.
-In the meantime the app ecosystem has grown used to life without this basic
-necessity, and many apps merrily ignore it even if it's there. This situation
-is improving, but in many circumstances using [transparent
-mode](@!urlTo("transparent.html")!@) is mandatory for testing Android apps.
-
-We used both an Asus Transformer Prime TF201 (Android 4.0.3) and a Nexus 4
-(Android 4.4.4) in the examples below - your device may differ, but the broad
-process should be similar.  On **emulated devices**, there are some [additional
-quirks](https://github.com/mitmproxy/mitmproxy/issues/204#issuecomment-32837093)
-to consider.
-
-
-## Getting the certificate onto the device
-
-The easiest way to get the certificate to the device is to use [the web
-app](@!urlTo("webapp.html")!@). In the rare cases where the web app doesn't
-work, you will need to get the __mitmproxy-ca-cert.cer__ file into the
-__/sdcard__ folder on the device (/sdcard/Download on older devices). This can
-be accomplished in a number of ways:
-
-- If you have the Android Developer Tools installed, you can use [__adb
-push__](http://developer.android.com/tools/help/adb.html).
-- Using a file transfer program like wget (installed on the Android device) to
-copy the file over.
-- Transfer the file using external media like an SD Card. 
-
-Once we have the certificate on the local disk, we need to import it into the
-list of trusted CAs. Go to Settings -> Security -> Credential Storage,
-and select "Install from storage":
-
-<img src="android-settingssecuritymenu.png"/>
-
-The certificate in /sdcard is automatically located and offered for
-installation. Installing the cert will delete the download file from the local
-disk.
-
-
-## Installing the certificate
-
-You should now see something like this (you may have to explicitly name the
-certificate):
-
-<img src="android-settingssecurityinstallca.png"/>
-
-Click OK, and you should then see the certificate listed in the Trusted
-Credentials store:
-
-<img src="android-settingssecurityuserinstalledca.png"/>
-

doc-src/certinstall/firefox.html

@@ -1,31 +0,0 @@
-## Get the certificate to the browser
-
-The easiest way to get the certificate to the browser is to use [the web
-app](@!urlTo("webapp.html")!@). If this fails, do the following:
-
-
-<ol class="tlist">
-    <li> If needed, copy the ~/.mitmproxy/mitmproxy-ca-cert.pem file to the target. </li>
-
-    <li>Open preferences, click on "Advanced", then select"Certificates":
-        <img src="@!urlTo('firefox3.jpg')!@"/>
-    </li>
-
-    <li> Click "View Certificates", "Import", and select the certificate file: 
-        <img src="@!urlTo('firefox3-import.jpg')!@"/>
-    </li>
-
-</ol>
-
-
-## Installing the certificate
-
-<ol class="tlist">
-    <li>Tick "Trust this CA to identify web sites", and click "Ok":
-        <img src="@!urlTo('firefox3-trust.jpg')!@"/>
-    </li>
-
-    <li> You should now see the mitmproxy certificate listed in the Authorities
-    tab.</li>
-</ol>
-

doc-src/certinstall/index.py

@@ -1,13 +0,0 @@
-from countershape import Page
-
-pages = [
-    Page("webapp.html", "Using the Web App"),
-    Page("firefox.html", "Firefox"),
-    Page("osx.html", "OSX"),
-    Page("windows7.html", "Windows 7"),
-    Page("ios.html", "IOS"),
-    Page("ios-simulator.html", "IOS Simulator"),
-    Page("android.html", "Android"),
-    Page("java.html", "Java"),
-    Page("mitm.it-error.html", "Error: No proxy configured"),
-]

doc-src/certinstall/ios-simulator.html

@@ -1,23 +0,0 @@
-
-How to install the __mitmproxy__ certificate authority in the IOS simulator:
-
-<ol class="tlist">
-
-    <li> First, check out the <a
-    href="https://github.com/ADVTOOLS/ADVTrustStore">ADVTrustStore</a> tool
-    from github.</li>
-
-    <li> Now, run the following command:
-
-    <pre class="terminal">./iosCertTrustManager.py -a ~/.mitmproxy/mitmproxy-ca-cert.pem</pre>
-
-    </li>
-
-</ol>
-
-
-Note that although the IOS simulator has its own certificate store, it shares
-the proxy settings of the host operating system. You will therefore to have
-configure your OSX host's proxy settings to use the mitmproxy instance you want
-to test with.
-

doc-src/certinstall/ios.html

@@ -1,27 +0,0 @@
-
-## Getting the certificate onto the device
-
-The easiest way to get the certificate to the device is to use [the web
-app](@!urlTo("webapp.html")!@). In the rare cases where the web app doesn't
-work, you will need to get the __mitmproxy-ca-cert.pem__ file to the device to
-install it. The easiest way to accomplish this is to set up the Mail app on the
-device, and to email it over as an attachment. Open the email, tap on the
-attachment, then proceed with the install.
-
-
-## Installing the certificate
-
-<ol class="tlist">
-    <li>You will be prompted to install a profile. Click "Install":
-
-    <img src="@!urlTo('ios-profile.png')!@"/></li>
-
-    <li>Accept the warning by clicking "Install" again:
-
-    <img src="@!urlTo('ios-warning.png')!@"/></li>
-
-    <li>The certificate should now be trusted:
-
-    <img src="@!urlTo('ios-installed.png')!@"/></li>
-
-</ol>

doc-src/certinstall/java.html

@@ -1,13 +0,0 @@
-
-You can add the mitmproxy certificates to the Java trust store using
-[keytool](http://docs.oracle.com/javase/6/docs/technotes/tools/solaris/keytool.html).
-On OSX, the required command looks like this:
-
-<pre class="terminal">
-sudo keytool -importcert -alias mitmproxy -storepass "password" \
--keystore /System/Library/Java/Support/CoreDeploy.bundle/Contents/Home/lib/security/cacerts \
--trustcacerts -file ~/.mitmproxy/mitmproxy-ca-cert.pem
-</pre>
-
-Note that your store password will (hopefully) be different from the one above.
-

doc-src/certinstall/mitm.it-error.html

@@ -1,5 +0,0 @@
-**Looks like you wanted to install the mitmproxy CA using the web app?**
-
-Unfortunately, there's been no mitmproxy instance on the wire that could have intercepted your request.
-Please configure your client to use mitmproxy and try again.<br>
-The request to <a href="http://mitm.it/">http://mitm.it/</a> must go through your mitmproxy instance.

doc-src/certinstall/osx.html

@@ -1,16 +0,0 @@
-
-How to install the __mitmproxy__ certificate authority in OSX:
-
-<ol class="tlist">
-
-    <li>Open Finder, and double-click on the mitmproxy-ca-cert.pem file.</li>
-
-    <li>You will be prompted to add the certificate. Click "Always Trust": 
-
-        <img src="@!urlTo('osx-addcert-alwaystrust.png')!@"/>
-    </li>
-
-    <li> You may be prompted for your password. You should now see the
-    mitmproxy cert listed under "Certificates".</li>
-</ol>
-

doc-src/certinstall/webapp.html

@@ -1,13 +0,0 @@
-
-By far the easiest way to install the mitmproxy certs is to use the built-in
-web app. To do this, start mitmproxy and configure your target device with the
-correct proxy settings. Now start a browser on the device, and visit the magic
-domain **mitm.it**. You should see something like this:
-
-<img src="@!urlTo("webapp.png")!@"></img>
-
-Just click on the relevant icon, and then follow the setup instructions
-for the platform you're on.
-
-Make sure you aren't using a bandwith optimizer (like Google's Data Compression
-Proxy on Chrome for Android) or the page will not load.

doc-src/certinstall/windows7.html

@@ -1,35 +0,0 @@
-
-How to install the __mitmproxy__ certificate authority in Windows 7:
-
-<ol class="tlist">
-
-    <li> The easiest way to get the certificate to the device is to use <a
-        href="@!urlTo("webapp.html")!@">the web app</a>. If this fails for some
-    reason, simply copy the ~/.mitmproxy/mitmproxy-ca-cert.p12 file to the
-    target system and double-click it.  </li>
-
-    <li> 
-        You should see a certificate import wizard: 
-
-        <img src="@!urlTo('win7-wizard.png')!@"/>
-    </li>
-
-    <li> 
-        Click "Next" until you're prompted for the certificate store:
-
-        <img src="@!urlTo('win7-certstore.png')!@"/>
-
-    </li>
-
-
-    <li> 
-        <p>Select "Place all certificates in the following store", and select "Trusted Root Certification Authorities":</p>
-
-        <img src="@!urlTo('win7-certstore-trustedroot.png')!@"/>
-
-    </li>
-
-    <li> Click "Next" and "Finish". </li>
-
-</ol>
-

doc-src/dev/architecture.html

@@ -1,8 +1,8 @@
-To give you a better understanding of how mitmproxy works, mitmproxy's high-level architecture is detailed
+To give you a better understanding of how mitmproxy works, mitmproxy's
-in the following graphic:
+high-level architecture is detailed in the following graphic:
 
-<img src="@!urlTo('schematics/architecture.png')!@">
+<img class="img-responsive" src="@!urlTo('schematics/architecture.png')!@">
 
 <a href="@!urlTo('schematics/architecture.pdf')!@">(architecture.pdf)</a>
 <p>Please don't refrain from asking any further
-questions on the mailing list, the IRC channel or the GitHub issue tracker.</p>
+questions on the mailing list, the IRC channel or the GitHub issue tracker.</p>

doc-src/dev/index.py

@@ -1,8 +1,8 @@
 from countershape import Page
 
 pages = [
-     Page("testing.html", "Testing"),
+    Page("testing.html", "Testing"),
-     Page("architecture.html", "Architecture"),
+    Page("architecture.html", "Architecture"),
-     Page("sslkeylogfile.html", "TLS Master Secrets"),
+    Page("sslkeylogfile.html", "TLS Master Secrets"),
-#    Page("addingviews.html", "Writing Content Views"),
+    #    Page("addingviews.html", "Writing Content Views"),
 ]

doc-src/features/index.py

@@ -16,4 +16,4 @@ pages = [
     Page("tcpproxy.html", "TCP Proxy"),
     Page("upstreamcerts.html", "Upstream Certs"),
     Page("upstreamproxy.html", "Upstream proxy mode"),
-]
+]

doc-src/features/passthrough.html

@@ -51,10 +51,10 @@ $ mitmdump -v
 127.0.0.1:50588: request
   -> CONNECT example.com:443 HTTP/1.1
 127.0.0.1:50588: Set new server address: example.com:443
-<span style="color: white">127.0.0.1:50588: serverconnect
+127.0.0.1:50588: serverconnect
-  -> example.com:443</span>
+  -> example.com:443
 ^C
-$ <span style="color: white">mitmproxy --ignore ^example\.com:443$</span>    
+$ mitmproxy --ignore ^example\.com:443$
 </pre>
 
 Here are some other examples for ignore patterns:
@@ -62,13 +62,13 @@ Here are some other examples for ignore patterns:
 # Exempt traffic from the iOS App Store (the regex is lax, but usually just works):
 --ignore apple.com:443  
 # "Correct" version without false-positives:
---ignore ^(.+\.)?apple\.com:443$ 
+--ignore '^(.+\.)?apple\.com:443$'
     
 # Ignore example.com, but not its subdomains:
---ignore ^example.com:
+--ignore '^example.com:'
 
 # Ignore everything but example.com and mitmproxy.org:
---ignore ^(?!example\.com)(?!mitmproxy\.org)
+--ignore '^(?!example\.com)(?!mitmproxy\.org)'
     
 # Transparent mode:
 --ignore 17\.178\.96\.59:443
@@ -81,4 +81,4 @@ Here are some other examples for ignore patterns:
 - [TCP Proxy](@!urlTo("tcpproxy.html")!@)
 - [Response Streaming](@!urlTo("responsestreaming.html")!@)
 
-[^explicithttp]: This stems from an limitation of explicit HTTP proxying: A single connection can be re-used for multiple target domains - a <code>GET http://example.com/</code> request may be followed by a <code>GET http://evil.com/</code> request on the same connection. If we start to ignore the connection after the first request, we would miss the relevant second one.
+[^explicithttp]: This stems from an limitation of explicit HTTP proxying: A single connection can be re-used for multiple target domains - a <code>GET http://example.com/</code> request may be followed by a <code>GET http://evil.com/</code> request on the same connection. If we start to ignore the connection after the first request, we would miss the relevant second one.

doc-src/howmitmproxy.html

@@ -26,7 +26,7 @@ This is a proxy GET request - an extended form of the vanilla HTTP GET request
 that includes a schema and host specification, and it includes all the
 information mitmproxy needs to proceed.
 
-<img src="explicit.png"/>
+<img class="img-responsive" src="explicit.png"/>
 
 <table class="table">
     <tbody>
@@ -84,7 +84,7 @@ attempts to MITM an SSL connection for analysis. Our answer to this conundrum
 is to become a trusted Certificate Authority ourselves. Mitmproxy includes a
 full CA implementation that generates interception certificates on the fly. To
 get the client to trust these certificates, we [register mitmproxy as a trusted
-CA with the device manually](@!urlTo("ssl.html")!@).
+CA with the device manually](@!urlTo("certinstall.html")!@).
 
 ## Complication 1: What's the remote hostname?
 
@@ -158,7 +158,7 @@ handshake. Luckily, this is almost never an issue in practice.
 
 Lets put all of this together into the complete explicitly proxied HTTPS flow.
 
-<img src="explicit_https.png"/>
+<img class="img-responsive" src="explicit_https.png"/>
 
 <table class="table">
     <tbody>
@@ -250,7 +250,7 @@ mitmproxy, this takes the form of a built-in set of
 that know how to talk to each platform's redirection mechanism.  Once we have
 this information, the process is fairly straight-forward.
 
-<img src="transparent.png"/>
+<img class="img-responsive" src="transparent.png"/>
 
 
 <table class="table">
@@ -296,7 +296,7 @@ transparently proxying HTTP, and explicitly proxying HTTPS. We use the routing
 mechanism to establish the upstream server address, and then proceed as for
 explicit HTTPS connections to establish the CN and SANs, and cope with SNI.
 
-<img src="transparent_https.png"/>
+<img class="img-responsive" src="transparent_https.png"/>
 
 
 <table class="table">

doc-src/index.py

@@ -2,7 +2,7 @@ import os
 import sys
 import datetime
 import countershape
-from countershape import Page, Directory, markup, model
+from countershape import Page, Directory, markup
 import countershape.template
 
 MITMPROXY_SRC = os.path.abspath(
@@ -15,10 +15,10 @@ ns.VERSION = version.VERSION
 
 if ns.options.website:
     ns.idxpath = "doc/index.html"
-    this.layout = countershape.Layout("_websitelayout.html")
 else:
     ns.idxpath = "index.html"
-    this.layout = countershape.Layout("_layout.html")
+
+this.layout = countershape.layout.FileLayout("_layout.html")
 
 ns.title = countershape.template.Template(None, "<h1>@!this.title!@</h1>")
 this.titlePrefix = "%s - " % version.NAMEVERSION
@@ -37,7 +37,7 @@ def mpath(p):
 def example(s):
     d = file(mpath(s)).read().rstrip()
     extemp = """<div class="example">%s<div class="example_legend">(%s)</div></div>"""
-    return extemp%(countershape.template.Syntax("py")(d), s)
+    return extemp % (countershape.template.Syntax("py")(d), s)
 
 
 ns.example = example
@@ -52,7 +52,8 @@ def nav(page, current, state):
     else:
         pre = "<li>"
     p = state.application.getPage(page)
-    return pre + '<a href="%s">%s</a></li>'%(model.UrlTo(page), p.title)
+    return pre + \
+        '<a href="%s">%s</a></li>' % (countershape.widgets.UrlTo(page), p.title)
 ns.nav = nav
 ns.navbar = countershape.template.File(None, "_nav.html")
 
@@ -60,6 +61,7 @@ ns.navbar = countershape.template.File(None, "_nav.html")
 pages = [
     Page("index.html", "Introduction"),
     Page("install.html", "Installation"),
+    Page("certinstall.html", "About Certificates"),
     Page("howmitmproxy.html", "How mitmproxy works"),
     Page("modes.html", "Modes of Operation"),
 
@@ -67,8 +69,6 @@ pages = [
     Page("mitmdump.html", "mitmdump"),
     Page("config.html", "configuration"),
 
-    Page("ssl.html", "Overview"),
-    Directory("certinstall"),
     Directory("scripting"),
     Directory("tutorials"),
     Page("transparent.html", "Overview"),

doc-src/install.html

@@ -1,40 +1,33 @@
+## On This Page
 
-## Installing from source
+* [Installation On Ubuntu](#docUbuntu)
+* [Installation On Mac OS X](#docOSX)
+* [Installation On Windows](#docWindows)
 
-The preferred way to install mitmproxy - whether you're installing the latest
+## <a id=docUbuntu></a>Installation On Ubuntu
-release or from source - is to use [pip](http://www.pip-installer.org/). If you
+
-don't already have pip on your system, you can find installation instructions
+Ubuntu comes with Python but we need to install pip, python-dev and several libraries.  This was tested on a fully patched installation of Ubuntu 14.04.
-[here](http://www.pip-installer.org/en/latest/installing.html).
 
 <pre class="terminal">
-pip install mitmproxy
+$ sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev
+
+$ sudo pip install mitmproxy
 </pre>
 
-If you also want to install the optional packages AMF, protobuf and CSS
+Once installation is complete you can run <a href="mitmproxy.html">mitmproxy</a> or <a href="mitmdump.html">mitmdump</a> from a terminal.
-content views, do this:
 
-<pre class="terminal">
+### Installation From Source
-pip install "mitmproxy[contentviews]"
+
-</pre>
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to get set up to contribute to the project,
+install the dependencies as you would for a regular mitmproxy installation (see previous section).
+Then see the <a href="https://github.com/mitmproxy/mitmproxy/blob/master/README.mkd#hacking">Hacking</a> section of the README on GitHub.
 
 
-## OSX
+## <a id=docOSX></a>Installation On Mac OS X
 
-The easiest way to get up and running on OSX is to download the pre-built
+The easiest way to get up and running on OSX is to download the pre-built binary packages from [mitmproxy.org](http://mitmproxy.org).
-binary packages from [mitmproxy.org](http://mitmproxy.org). If you still want
-to install using pip, there are a few things to keep in mind:
 
-- If you're running a Python interpreter installed with homebrew (or similar),
+There are a few bits of customization you might want to do to make mitmproxy comfortable to use on OSX. The default color scheme is optimized for a dark background terminal, but you can select a palette for a light terminal background with the --palette option. You can use the OSX <b>open</b> program to create a simple and effective <b>~/.mailcap</b> file to view request and response bodies:
-you may have to install some dependencies by hand. 
-- Make sure that XCode is installed from the App Store, and that the
-command-line tools have been downloaded (XCode/Preferences/Downloads).
-
-There are a few bits of customization you might want to do to make mitmproxy
-comfortable to use on OSX. The default color scheme is optimized for a dark
-background terminal, but you can select a palette for a light terminal
-background with the --palette option. You can use the OSX <b>open</b> program
-to create a simple and effective <b>~/.mailcap</b> file to view request and
-response bodies:
 
 <pre class="terminal">
 application/*; /usr/bin/open -Wn %s
@@ -43,17 +36,44 @@ image/*; /usr/bin/open -Wn %s
 video/*; /usr/bin/open -Wn %s
 </pre>
 
+Once installation is complete you can run <a href="mitmproxy.html">mitmproxy</a> or <a href="mitmdump.html">mitmdump</a> from a terminal.
 
-## Ubuntu
+### Installation From Source
 
-On Ubuntu, you will need the following native packages to install mitmproxy
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to get set up to contribute to the project, ithere are a few OS X specific things to keep in mind.
-from source:
 
-- build-essential
+- Make sure that XCode is installed from the App Store, and that the command-line tools have been downloaded (XCode/Preferences/Downloads).
-- python-dev
+- If you're running a Python interpreter installed with homebrew (or similar), you may have to install some dependencies by hand. 
-- libffi-dev
+
-- libssl-dev
+Then see the <a href="https://github.com/mitmproxy/mitmproxy/blob/master/README.mkd#hacking">Hacking</a> section of the README on GitHub.
-- libxml2-dev
+
-- libxslt1-dev
+## <a id=docWindows></a>Installation On Windows
+
+Please note that mitmdump is the only component of mitmproxy that is supported on Windows at the moment.
+There is no interactive user interface on Windows.
+
+
+First, install the latest version of Python 2.7 from the <a href="https://www.python.org/downloads/windows/">Python website</a>.
+If you already have an older version of Python 2.7 installed, make sure to install <a href="https://pip.pypa.io/en/latest/installing.html">pip</a>
+(pip is included in Python 2.7.9+ by default).
+
+Next, add Python and the Python Scripts directory to your <strong>PATH</strong> variable. You can do this easily by running the following in powershell:
+
+<pre class="terminal">
+[Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27\;C:\Python27\Scripts\", "User")
+</pre>
+
+
+Now, you can install mitmproxy by running
+
+<pre class="terminal">
+pip install mitmproxy
+</pre>
+
+Once the installation is complete, you can run <a href="mitmdump.html">mitmdump</a> from a command prompt.
+
+### Installation From Source
+
+If you would like to install mitmproxy directly from the master branch on GitHub or would like to get set up to contribute to the project, install Python as outlined above, then see the <a href="https://github.com/mitmproxy/mitmproxy/blob/master/README.mkd#hacking">Hacking</a> section of the README on GitHub.
 
 

doc-src/mitmproxy.html

@@ -7,13 +7,13 @@ documentation from any __mitmproxy__ screen.
 
 ## Flow list
 
-The flow list shows an index of captured flows in chronological order. 
+The flow list shows an index of captured flows in chronological order.
 
-<img src="@!urlTo('screenshots/mitmproxy.png')!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy.png')!@"/>
 
 - __1__: A GET request, returning a 302 Redirect response.
 - __2__: A GET request, returning 16.75kb of text/html data.
-- __3__: A replayed request. 
+- __3__: A replayed request.
 - __4__: Intercepted flows are indicated with orange text. The user may edit
 these flows, and then accept them (using the _a_ key) to continue. In this
 case, the request has been intercepted on the way to the server.
@@ -32,7 +32,7 @@ interfaces.
 
 The __Flow View__ lets you inspect and manipulate a single flow:
 
-<img src="@!urlTo('screenshots/mitmproxy-flowview.png')!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-flowview.png')!@"/>
 
 - __1__: Flow summary.
 - __2__: The Request/Response tabs, showing you which part of the flow you are
@@ -53,11 +53,11 @@ using the _m_ key.
 Much of the data that we'd like to interact with in mitmproxy is structured.
 For instance, headers, queries and form data can all be thought of as a list of
 key/value pairs. Mitmproxy has a built-in editor that lays this type of data
-out in a grid for easy manipulation. 
+out in a grid for easy manipulation.
 
 At the moment, the Grid Editor is used in four parts of mitmproxy:
 
-- Editing request or response headers (_e_ for edit, then _h_ for headers in flow view) 
+- Editing request or response headers (_e_ for edit, then _h_ for headers in flow view)
 - Editing a query string (_e_ for edit, then _q_ for query in flow view)
 - Editing a URL-encoded form (_e_ for edit, then _f_ for form in flow view)
 - Editing replacement patterns (_R_ globally)
@@ -65,13 +65,13 @@ At the moment, the Grid Editor is used in four parts of mitmproxy:
 If there is is no data, an empty editor will be started to let you add some.
 Here is the editor showing the headers from a request:
 
-<img src="@!urlTo('screenshots/mitmproxy-kveditor.png')!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-kveditor.png')!@"/>
 
 To edit, navigate to the key or value you want to modify using the arrow or vi
 navigation keys, and press enter. The background color will change to show that
 you are in edit mode for the specified field:
 
-<img src="@!urlTo('screenshots/mitmproxy-kveditor-editmode.png')!@"/>
+<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-kveditor-editmode.png')!@"/>
 
 Modify the field as desired, then press escape to exit edit mode when you're
 done. You can also add a row (_a_ key), delete a row (_d_ key), spawn an
@@ -83,12 +83,12 @@ help (_?_ key) for more.
 
 __mitmproxy__'s interception functionality lets you pause an HTTP request or
 response, inspect and modify it, and then accept it to send it on to the server
-or client. 
+or client.
 
 
 ### 1: Set an interception pattern
 
-<img src="@!urlTo('mitmproxy-intercept-filt.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-filt.png')!@"/>
 
 We press _i_ to set an interception pattern. In this case, the __~q__ filter
 pattern tells __mitmproxy__ to intercept all requests. For complete filter
@@ -97,19 +97,19 @@ document, or the built-in help function in __mitmproxy__.
 
 ### 2: Intercepted connections are indicated with orange text:
 
-<img src="@!urlTo('mitmproxy-intercept-mid.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-mid.png')!@"/>
 
 ### 3: You can now view and modify the request:
 
-<img src="@!urlTo('mitmproxy-intercept-options.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-options.png')!@"/>
 
 In this case, we viewed the request by selecting it, pressed _e_ for "edit"
 and _m_ for "method" to change the HTTP request method.
 
 ### 4: Accept the intercept to continue:
 
-<img src="@!urlTo('mitmproxy-intercept-result.png')!@"/>
+<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-result.png')!@"/>
 
 Finally, we press _a_ to accept the modified request, which is then sent on to
 the server. In this case, we changed the request from an HTTP GET to
-OPTIONS, and Google's server has responded with a 405 "Method not allowed". 
+OPTIONS, and Google's server has responded with a 405 "Method not allowed".

doc-src/modes.html

@@ -9,7 +9,7 @@ variety of scenarios:
 
 Now, which one should you pick? Use this flow chart:
 
-<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
+<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
 
 <div class="page-header">
     <h1>Regular Proxy</h1>
@@ -31,7 +31,7 @@ these cases, you need to use mitmproxy's transparent mode.
 
 If you are proxying an external device, your network will probably look like this:
 
-<img src="@!urlTo('schematics/proxy-modes-regular.png')!@">
+<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-regular.png')!@">
 
 The square brackets signify the source and destination IP addresses. Your
 client explicitly connects to mitmproxy and mitmproxy explicitly connects
@@ -48,7 +48,7 @@ below, a machine running mitmproxy has been inserted between the router and
 the internet:
 
 <a href="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
 </a>
 
 The square brackets signify the source and destination IP addresses. Round
@@ -60,7 +60,7 @@ remove the target information, leaving mitmproxy unable to determine the real
 destination.
 
 <a href="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
 
 <h2>Common Configurations</h2>
 
@@ -79,7 +79,7 @@ intact, is to simply configure the client with the mitmproxy box as the
 default gateway.
 
 <a href="@!urlTo('schematics/proxy-modes-transparent-2.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
 
 In this scenario, we would:
 
@@ -98,7 +98,7 @@ for your device.
 
 Setting the custom gateway on clients can be automated by serving the settings
 out to clients over DHCP. This lets set up an interception network where all
-clients are proxied automatically, which can save time and effort. 
+clients are proxied automatically, which can save time and effort.
 
 
 <div class="well">
@@ -141,7 +141,7 @@ packet filter you're using. In most cases, the configuration will look like
 this:
 
 <a href="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
 </a>
 
 
@@ -154,7 +154,7 @@ Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal
 HTTP server:
 
 <a href="@!urlTo('schematics/proxy-modes-reverse.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
 </a>
 
 There are various use-cases:
@@ -215,8 +215,8 @@ appliance, you can use mitmproxy's upstream mode. In upstream mode, all
 requests are unconditionally transferred to an upstream proxy of your choice.
 
 <a href="@!urlTo('schematics/proxy-modes-upstream.png')!@">
-    <img src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
+    <img class="img-responsive" src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
 
 mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy
 mode. You could in theory chain multiple mitmproxy instances in a row, but
-that doesn't make any sense in practice (i.e. outside of our tests). 
+that doesn't make any sense in practice (i.e. outside of our tests).

doc-src/scripting/inlinescripts.html

@@ -122,12 +122,11 @@ The main classes you will deal with in writing mitmproxy scripts are:
         <td> A handle for interacting with mitmproxy's from within scripts.</td>
     </tr>
     <tr>
-        <th>libmproxy.flow.ODict</th>
+        <th>netlib.odict.ODict</th>
 
         <td>A dictionary-like object for managing sets of key/value data. There
-            is also a variant called CaselessODict that ignores key case for some
+        is also a variant called ODictCaseless that ignores key case for some
-            calls (used mainly for headers).
+        calls (used mainly for headers).</td>
-        </td>
     </tr>
     <tr>
         <th>netlib.certutils.SSLCert</th>

doc-src/ssl.html

@@ -1,99 +0,0 @@
-
-The first time __mitmproxy__ or __mitmdump__ is run, a set of certificate files
-for the mitmproxy Certificate Authority are created in the config directory
-(~/.mitmproxy by default). This CA is used for on-the-fly generation of dummy
-certificates for SSL interception. Since your browser won't trust the
-__mitmproxy__ CA out of the box (and rightly so), you will see an SSL cert
-warning every time you visit a new SSL domain through __mitmproxy__. When
-you're testing a single site through a browser, just accepting the bogus SSL
-cert manually is not too much trouble, but there are a many circumstances where
-you will want to configure your testing system or browser to trust the
-__mitmproxy__ CA as a signing root authority.
-
-
-CA and cert files
------------------
-
-The files created by mitmproxy in the .mitmproxy directory are as follows: 
-
-<table class="table">
-    <tr>
-        <td class="nowrap">mitmproxy-ca.pem</td>
-        <td>The private key and certificate in PEM format.</td>
-    </tr>
-    <tr>
-        <td class="nowrap">mitmproxy-ca-cert.pem</td>
-        <td>The certificate in PEM format. Use this to distribute to most
-        non-Windows platforms.</td>
-    </tr>
-    <tr>
-        <td class="nowrap">mitmproxy-ca-cert.p12</td>
-        <td>The certificate in PKCS12 format. For use on Windows.</td>
-    </tr>
-    <tr>
-        <td class="nowrap">mitmproxy-ca-cert.cer</td>
-        <td>Same file as .pem, but with an extension expected by some Android
-        devices.</td>
-    </tr>
-</table>
-    
-
-Using a custom certificate
---------------------------
-
-You can use your own certificate by passing the <kbd>--cert</kbd> option to mitmproxy. mitmproxy then uses the provided
-certificate for interception of the specified domains instead of generating a cert signed by its own CA.
-
-The certificate file is expected to be in the PEM format.
-You can include intermediary certificates right below your leaf certificate, so that you PEM file roughly looks like
-this:
-
-<pre>
------BEGIN PRIVATE KEY-----
-&lt;private key&gt;
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-&lt;cert&gt;
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-&lt;intermediary cert (optional)&gt;
------END CERTIFICATE-----
-</pre>
-
-For example, you can generate a certificate in this format using these instructions:
-
-<pre class="terminal">
-> openssl genrsa -out cert.key 8192
-> openssl req -new -x509 -key cert.key -out cert.crt
-    (Specify the mitm domain as Common Name, e.g. *.google.com)
-> cat cert.key cert.crt > cert.pem
-> mitmproxy --cert=cert.pem
-</pre>
-
-Using a client side certificate
-------------------------------------
-You can use a client certificate by passing the <kbd>--client-certs DIRECTORY</kbd> option to mitmproxy.
-If you visit example.org, mitmproxy looks for a file named example.org.pem in the specified directory
-and uses this as the client cert. The certificate file needs to be in the PEM format and should contain
-both the unencrypted private key as well as the certificate.
-
-
-Using a custom certificate authority
-------------------------------------
-
-By default, mitmproxy will (generate and) use <samp>~/.mitmproxy/mitmproxy-ca.pem</samp> as the default certificate
-authority to generate certificates for all domains for which no custom certificate is provided (see above).
-You can use your own certificate authority by passing the <kbd>--confdir</kbd> option to mitmproxy.
-mitmproxy will then look for <samp>mitmproxy-ca.pem</samp> in the specified directory. If no such file exists,
-it will be generated automatically.
-
-Installing the mitmproxy CA
----------------------------
-
-* [Firefox](@!urlTo("certinstall/firefox.html")!@)
-* [OSX](@!urlTo("certinstall/osx.html")!@)
-* [Windows 7](@!urlTo("certinstall/windows7.html")!@)
-* [iPhone/iPad](@!urlTo("certinstall/ios.html")!@)
-* [IOS Simulator](@!urlTo("certinstall/ios-simulator.html")!@)
-* [Android](@!urlTo("certinstall/android.html")!@)
-

doc-src/transparent/linux.html

@@ -3,7 +3,7 @@ achieve transparent mode.
 
 <ol class="tlist">
 
-    <li> <a href="@!urlTo('ssl.html')!@">Install the mitmproxy
+    <li> <a href="@!urlTo('certinstall.html')!@">Install the mitmproxy
     certificates on the test device</a>. </li>
 
     <li> Enable IP forwarding:
@@ -31,7 +31,7 @@ achieve transparent mode.
 
 <pre class="terminal">iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080
 iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080</pre>
-    
+
     </li>
 
     <li> Fire up mitmproxy. You probably want a command like this:

doc-src/transparent/osx.html

@@ -7,7 +7,7 @@ OSX.
 
 <ol class="tlist">
 
-    <li> <a href="@!urlTo('ssl.html')!@">Install the mitmproxy
+    <li> <a href="@!urlTo('certinstall.html')!@">Install the mitmproxy
     certificates on the test device</a>. </li>
 
     <li> Enable IP forwarding:
@@ -76,6 +76,3 @@ want to intercept your OSX traffic, you should use an external host to run
 mitmproxy. None the less, pf is flexible to cater for a range of creative
 possibilities, like intercepting traffic emanating from VMs.  See the
 **pf.conf** man page for more.
-
-
-

doc-src/tutorials/gamecenter.html

@@ -2,11 +2,9 @@
 ## The setup
 
 In this tutorial, I'm going to show you how simple it is to creatively
-interfere with Apple Game Center traffic using mitmproxy. To set things up, I
+interfere with Apple Game Center traffic using mitmproxy. To set things up,
-registered my mitmproxy CA certificate with my iPhone - there's a [step by step
+install the [mitmproxy root certificate](@!urlTo("certinstall.html")!@). Then
-set of instructions](@!urlTo("certinstall/ios.html")!@) elsewhere in this manual. I then
+start mitmproxy on your desktop, and confige the iPhone to use it as a proxy.
-started mitmproxy on my desktop, and configured the iPhone to use it as a
-proxy. 
 
 
 ## Taking a look at the Game Center traffic
@@ -14,17 +12,17 @@ proxy.
 Lets take a first look at the Game Center traffic. The game I'll use in this
 tutorial is [Super Mega
 Worm](http://itunes.apple.com/us/app/super-mega-worm/id388541990?mt=8) - a
-great little retro-apocalyptic sidescroller for the iPhone: 
+great little retro-apocalyptic sidescroller for the iPhone:
 
 <center>
-    <img src="@!urlTo('tutorials/supermega.png')!@"/>
+    <img class="img-responsive" src="@!urlTo('tutorials/supermega.png')!@"/>
 </center>
 
 After finishing a game (take your time), watch the traffic flowing through
 mitmproxy:
 
 <center>
-    <img src="@!urlTo('tutorials/one.png')!@"/>
+    <img class="img-responsive" src="@!urlTo('tutorials/one.png')!@"/>
 </center>
 
 We see a bunch of things we might expect - initialisation, the retrieval of
@@ -91,7 +89,7 @@ now fire up. Lets bump the score up to something a bit more ambitious:
 </plist>
 <!--(end)-->
 
-Save the file and exit your editor. 
+Save the file and exit your editor.
 
 The final step is to replay this modified request. Simply press __r__ for
 replay.
@@ -99,11 +97,11 @@ replay.
 ## The glorious result and some intrigue
 
 <center>
-    <img src="@!urlTo('tutorials/leaderboard.png')!@"/>
+    <img class="img-responsive" src="@!urlTo('tutorials/leaderboard.png')!@"/>
 </center>
 
 And that's it - according to the records, I am the greatest Super Mega Worm
-player of all time. 
+player of all time.
 
 There's a curious addendum to this tale. When I first wrote this tutorial, all
 the top competitors' scores were the same: 2,147,483,647 (this is no longer the
@@ -119,4 +117,3 @@ phone, then 2^31-1 might well be the maximum score you could get. Then again,
 if the game itself stores its score in a signed 32-bit int, you could get the
 same score through perfect play, effectively beating the game. So, which is it
 in this case? I'll leave that for you to decide.
-

doc-src/tutorials/index.py

@@ -4,4 +4,4 @@ pages = [
     Page("30second.html", "Client playback: a 30 second example"),
     Page("gamecenter.html", "Setting highscores on Apple's GameCenter"),
     Page("transparent-dhcp.html", "Transparently proxify virtual machines")
-]
+]

doc-src/tutorials/transparent-dhcp.html

@@ -1,27 +1,27 @@
 This walkthrough illustrates how to set up transparent proxying with mitmproxy. We use VirtualBox VMs with an Ubuntu proxy machine in this example, but the general principle can be applied to other setups.
 
-1.  **Configure VirtualBox Network Adapters for the proxy machine**  
+1.  **Configure VirtualBox Network Adapters for the proxy machine**
-    The network setup is simple:  `internet <--> proxy vm <--> (virtual) internal network`.  
+    The network setup is simple:  `internet <--> proxy vm <--> (virtual) internal network`.
-    For the proxy machine, *eth0* represents the outgoing network. *eth1* is connected to the internal network that will be proxified, using a static ip (192.168.3.1).  
+    For the proxy machine, *eth0* represents the outgoing network. *eth1* is connected to the internal network that will be proxified, using a static ip (192.168.3.1).
     <hr>VirtualBox configuration:
-    <img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth0.png')!@"/><br><br>
+    <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth0.png')!@"/><br><br>
-    <img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth1.png')!@"/>  
+    <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth1.png')!@"/>
     <br>Proxy VM:
-    <img src="@!urlTo('tutorials/transparent-dhcp/step1_proxy.png')!@"/>  
+    <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_proxy.png')!@"/>
     <hr>
-2.  **Configure DHCP and DNS**  
+2.  **Configure DHCP and DNS**
-    We use dnsmasq to provide DHCP and DNS in our internal network. 
+    We use dnsmasq to provide DHCP and DNS in our internal network.
-    Dnsmasq is a lightweight server designed to provide DNS (and optionally DHCP and TFTP) services to a small-scale 
+    Dnsmasq is a lightweight server designed to provide DNS (and optionally DHCP and TFTP) services to a small-scale
-    network.  
+    network.
 
-    -   Before we get to that, we need to fix some Ubuntu quirks: 
+    -   Before we get to that, we need to fix some Ubuntu quirks:
-        **Ubuntu >12.04** runs an internal dnsmasq instance (listening on loopback only) by default 
+        **Ubuntu >12.04** runs an internal dnsmasq instance (listening on loopback only) by default
         <a href="https://www.stgraber.org/2012/02/24/dns-in-ubuntu-12-04/">[1]</a>. For our use case, this needs to be
-        disabled by changing <br>`dns=dnsmasq` to `#dns=dnsmasq` in */etc/NetworkManager/NetworkManager.conf*  
+        disabled by changing <br>`dns=dnsmasq` to `#dns=dnsmasq` in */etc/NetworkManager/NetworkManager.conf*
         and running `sudo restart network-manager` afterwards.
-    -   Now, dnsmasq can be be installed and configured:  
+    -   Now, dnsmasq can be be installed and configured:
-        `sudo apt-get install dnsmasq`  
+        `sudo apt-get install dnsmasq`
-        Replace */etc/dnsmasq.conf* with the following configuration:  
+        Replace */etc/dnsmasq.conf* with the following configuration:
         <pre>\# Listen for DNS requests on the internal network
         interface=eth1
         \# Act as a DHCP server, assign IP addresses to clients
@@ -30,15 +30,15 @@ This walkthrough illustrates how to set up transparent proxying with mitmproxy.
         dhcp-option=option:router,192.168.3.1
         dhcp-option=option:dns-server,192.168.3.1
         </pre>
-        Apply changes:  
+        Apply changes:
         `sudo service dnsmasq restart`
         <hr>
         Your proxied machine's network settings should now look similar to this:
-        <img src="@!urlTo('tutorials/transparent-dhcp/step2_proxied_vm.png')!@"/>
+        <img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step2_proxied_vm.png')!@"/>
         <hr>
 
-3.  **Set up traffic redirection to mitmproxy**  
+3.  **Set up traffic redirection to mitmproxy**
-    To redirect traffic to mitmproxy, we need to add two iptables rules:  
+    To redirect traffic to mitmproxy, we need to add two iptables rules:
     <pre class="terminal">
     iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 \
         -j REDIRECT --to-port 8080
@@ -46,9 +46,8 @@ This walkthrough illustrates how to set up transparent proxying with mitmproxy.
         -j REDIRECT --to-port 8080
     </pre>
 
-4.  If required, <a href="@!urlTo('ssl.html')!@">install the mitmproxy
+4.  If required, <a href="@!urlTo('certinstall.html')!@">install the mitmproxy
     certificates on the test device</a>.
 
-5.  Finally, we can run <code>mitmproxy -T</code>.  
+5.  Finally, we can run <code>mitmproxy -T</code>.
     The proxied machine cannot to leak any data outside of HTTP or DNS requests.
-

examples/README

@@ -3,6 +3,8 @@ add_header.py             Simple script that just adds a header to every request
 change_upstream_proxy.py  Dynamically change the upstream proxy
 dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
 dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
+filt.py                   Use mitmproxy's filter expressions in your script.
+flowwriter.py             Only write selected flows into a mitmproxy dumpfile.
 iframe_injector.py        Inject configurable iframe into pages.
 modify_form.py            Modify all form submissions to add a parameter.
 modify_querystring.py     Modify all query strings to add a parameters.

examples/add_header.py

@@ -1,2 +1,2 @@
 def response(context, flow):
-    flow.response.headers["newheader"] = ["foo"]
+    flow.response.headers["newheader"] = ["foo"]

examples/change_upstream_proxy.py

@@ -1,10 +1,13 @@
 # This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy
 # in upstream proxy mode.
 #
-# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s "change_upstream_proxy.py host"
+# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s
+# "change_upstream_proxy.py host"
 from libmproxy.protocol.http import send_connect_request
 
 alternative_upstream_proxy = ("localhost", 8082)
+
+
 def should_redirect(flow):
     return flow.request.host == "example.com"
 
@@ -15,7 +18,12 @@ def request(context, flow):
         # If you want to change the target server, you should modify flow.request.host and flow.request.port
         # flow.live.change_server should only be used by inline scripts to change the upstream proxy,
         # unless you are sure that you know what you are doing.
-        server_changed = flow.live.change_server(alternative_upstream_proxy, persistent_change=True)
+        server_changed = flow.live.change_server(
+            alternative_upstream_proxy,
+            persistent_change=True)
         if flow.request.scheme == "https" and server_changed:
-            send_connect_request(flow.live.c.server_conn, flow.request.host, flow.request.port)
+            send_connect_request(
+                flow.live.c.server_conn,
+                flow.request.host,
+                flow.request.port)
             flow.live.c.establish_ssl(server=True)

examples/dns_spoofing.py

@@ -25,11 +25,13 @@ mitmproxy -p 443 -R https2http://localhost:8000
 def request(context, flow):
     if flow.client_conn.ssl_established:
         # TLS SNI or Host header
-        flow.request.host = flow.client_conn.connection.get_servername() or flow.request.pretty_host(hostheader=True)
+        flow.request.host = flow.client_conn.connection.get_servername(
+        ) or flow.request.pretty_host(hostheader=True)
 
-        # If you use a https2http location as default destination, these attributes need to be corrected as well:
+        # If you use a https2http location as default destination, these
+        # attributes need to be corrected as well:
         flow.request.port = 443
         flow.request.scheme = "https"
     else:
         # Host header
-        flow.request.host = flow.request.pretty_host(hostheader=True)
+        flow.request.host = flow.request.pretty_host(hostheader=True)

examples/dup_and_replay.py

@@ -1,4 +1,4 @@
 def request(context, flow):
     f = context.duplicate_flow(flow)
     f.request.path = "/changed"
-    context.replay_request(f)
+    context.replay_request(f)

examples/filt.py

@@ -0,0 +1,16 @@
+# This scripts demonstrates how to use mitmproxy's filter pattern in inline scripts.
+# Usage: mitmdump -s "filt.py FILTER"
+
+from libmproxy import filt
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError("Usage: -s 'filt.py FILTER'")
+    context.filter = filt.parse(argv[1])
+
+
+def response(context, flow):
+    if flow.match(context.filter):
+        print("Flow matches filter:")
+        print(flow)

examples/flowbasic

@@ -36,7 +36,8 @@ class MyMaster(flow.FlowMaster):
 
 config = proxy.ProxyConfig(
     port=8080,
-    cadir="~/.mitmproxy/"  # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
+    # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
+    cadir="~/.mitmproxy/"
 )
 state = flow.State()
 server = ProxyServer(config)

examples/flowwriter.py

@@ -0,0 +1,20 @@
+import random
+import sys
+
+from libmproxy.flow import FlowWriter
+
+
+def start(context, argv):
+    if len(argv) != 2:
+        raise ValueError('Usage: -s "flowriter.py filename"')
+
+    if argv[1] == "-":
+        f = sys.stdout
+    else:
+        f = open(argv[1], "wb")
+    context.flow_writer = FlowWriter(f)
+
+
+def response(context, flow):
+    if random.choice([True, False]):
+        context.flow_writer.add(flow)

examples/har_extractor.py

@@ -83,7 +83,8 @@ def response(context, flow):
         # Calculate the connect_time for this server_conn. Afterwards add it to
         # seen list, in order to avoid the connect_time being present in entries
         # that use an existing connection.
-        connect_time = flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start
+        connect_time = flow.server_conn.timestamp_tcp_setup - \
+            flow.server_conn.timestamp_start
         context.seen_server.add(flow.server_conn)
 
         if flow.server_conn.timestamp_ssl_setup is not None:
@@ -91,7 +92,8 @@ def response(context, flow):
             # the start of the successful tcp setup and the successful ssl
             # setup. If  no ssl setup has been made it is left as -1 since it
             # doesn't apply to this connection.
-            ssl_time = flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup
+            ssl_time = flow.server_conn.timestamp_ssl_setup - \
+                flow.server_conn.timestamp_tcp_setup
 
     # Calculate the raw timings from the different timestamps present in the
     # request and response object. For lack of a way to measure it dns timings
@@ -110,7 +112,8 @@ def response(context, flow):
 
     # HAR timings are integers in ms, so we have to re-encode the raw timings to
     # that format.
-    timings = dict([(key, int(1000 * value)) for key, value in timings_raw.iteritems()])
+    timings = dict([(key, int(1000 * value))
+                    for key, value in timings_raw.iteritems()])
 
     # The full_time is the sum of all timings. Timings set to -1 will be ignored
     # as per spec.
@@ -119,20 +122,27 @@ def response(context, flow):
         if item > -1:
             full_time += item
 
-    started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, tz=utc).isoformat()
+    started_date_time = datetime.fromtimestamp(
+        flow.request.timestamp_start,
+        tz=utc).isoformat()
 
-    request_query_string = [{"name": k, "value": v} for k, v in flow.request.get_query()]
+    request_query_string = [{"name": k, "value": v}
+                            for k, v in flow.request.get_query()]
     request_http_version = ".".join([str(v) for v in flow.request.httpversion])
     # Cookies are shaped as tuples by MITMProxy.
-    request_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.request.get_cookies() or {}).iteritems()]
+    request_cookies = [{"name": k.strip(), "value": v[0]}
+                       for k, v in (flow.request.get_cookies() or {}).iteritems()]
     request_headers = [{"name": k, "value": v} for k, v in flow.request.headers]
     request_headers_size = len(str(flow.request.headers))
     request_body_size = len(flow.request.content)
 
-    response_http_version = ".".join([str(v) for v in flow.response.httpversion])
+    response_http_version = ".".join(
+        [str(v) for v in flow.response.httpversion])
     # Cookies are shaped as tuples by MITMProxy.
-    response_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.response.get_cookies() or {}).iteritems()]
+    response_cookies = [{"name": k.strip(), "value": v[0]}
-    response_headers = [{"name": k, "value": v} for k, v in flow.response.headers]
+                        for k, v in (flow.response.get_cookies() or {}).iteritems()]
+    response_headers = [{"name": k, "value": v}
+                        for k, v in flow.response.headers]
     response_headers_size = len(str(flow.response.headers))
     response_body_size = len(flow.response.content)
     response_body_decoded_size = len(flow.response.get_decoded_content())
@@ -140,33 +150,43 @@ def response(context, flow):
     response_mime_type = flow.response.headers.get_first('Content-Type', '')
     response_redirect_url = flow.response.headers.get_first('Location', '')
 
-    entry = HAR.entries({"startedDateTime": started_date_time,
+    entry = HAR.entries(
-                         "time": full_time,
+        {
-                         "request": {"method": flow.request.method,
+            "startedDateTime": started_date_time,
-                                     "url": flow.request.url,
+            "time": full_time,
-                                     "httpVersion": request_http_version,
+            "request": {
-                                     "cookies": request_cookies,
+                "method": flow.request.method,
-                                     "headers": request_headers,
+                "url": flow.request.url,
-                                     "queryString": request_query_string,
+                "httpVersion": request_http_version,
-                                     "headersSize": request_headers_size,
+                "cookies": request_cookies,
-                                     "bodySize": request_body_size, },
+                "headers": request_headers,
-                         "response": {"status": flow.response.code,
+                "queryString": request_query_string,
-                                      "statusText": flow.response.msg,
+                "headersSize": request_headers_size,
-                                      "httpVersion": response_http_version,
+                "bodySize": request_body_size,
-                                      "cookies": response_cookies,
+            },
-                                      "headers": response_headers,
+            "response": {
-                                      "content": {"size": response_body_size,
+                "status": flow.response.code,
-                                                  "compression": response_body_compression,
+                "statusText": flow.response.msg,
-                                                  "mimeType": response_mime_type},
+                "httpVersion": response_http_version,
-                                      "redirectURL": response_redirect_url,
+                "cookies": response_cookies,
-                                      "headersSize": response_headers_size,
+                "headers": response_headers,
-                                      "bodySize": response_body_size, },
+                "content": {
-                         "cache": {},
+                    "size": response_body_size,
-                         "timings": timings, })
+                    "compression": response_body_compression,
+                    "mimeType": response_mime_type},
+                "redirectURL": response_redirect_url,
+                "headersSize": response_headers_size,
+                "bodySize": response_body_size,
+            },
+            "cache": {},
+            "timings": timings,
+        })
 
     # If the current url is in the page list of context.HARLog or does not have
     # a referrer we add it as a new pages object.
-    if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get('Referer', None) is None:
+    if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get(
+            'Referer',
+            None) is None:
         page_id = context.HARLog.create_page_id()
         context.HARLog.add(
             HAR.pages({

examples/iframe_injector.py

@@ -16,7 +16,12 @@ def response(context, flow):
     with decoded(flow.response):  # Remove content encoding (gzip, ...)
         html = BeautifulSoup(flow.response.content)
         if html.body:
-            iframe = html.new_tag("iframe", src=context.iframe_url, frameborder=0, height=0, width=0)
+            iframe = html.new_tag(
+                "iframe",
+                src=context.iframe_url,
+                frameborder=0,
+                height=0,
+                width=0)
             html.body.insert(0, iframe)
             flow.response.content = str(html)
-            context.log("Iframe inserted.")
+            context.log("Iframe inserted.")

examples/ignore_websocket.py

@@ -24,11 +24,13 @@ def done(context):
     HTTPRequest._headers_to_strip_off.append("Connection")
     HTTPRequest._headers_to_strip_off.append("Upgrade")
 
+
 @concurrent
 def response(context, flow):
-    if flow.response.headers.get_first("Connection", None) == "Upgrade":
+    value = flow.response.headers.get_first("Connection", None)
+    if value and value.upper() == "UPGRADE":
         # We need to send the response manually now...
         flow.client_conn.send(flow.response.assemble())
         # ...and then delegate to tcp passthrough.
         TCPHandler(flow.live.c, log=False).handle_messages()
-        flow.reply(KILL)
+        flow.reply(KILL)

examples/mitmproxywrapper.py

@@ -14,23 +14,29 @@ import contextlib
 import os
 import sys
 
+
 class Wrapper(object):
-    
+
     def __init__(self, port, extra_arguments=None):
         self.port = port
         self.extra_arguments = extra_arguments
 
     def run_networksetup_command(self, *arguments):
-        return subprocess.check_output(['sudo', 'networksetup'] + list(arguments))
+        return subprocess.check_output(
+            ['sudo', 'networksetup'] + list(arguments))
 
     def proxy_state_for_service(self, service):
-        state = self.run_networksetup_command('-getwebproxy', service).splitlines()
+        state = self.run_networksetup_command(
+            '-getwebproxy',
+            service).splitlines()
         return dict([re.findall(r'([^:]+): (.*)', line)[0] for line in state])
 
     def enable_proxy_for_service(self, service):
         print('Enabling proxy on {}...'.format(service))
         for subcommand in ['-setwebproxy', '-setsecurewebproxy']:
-            self.run_networksetup_command(subcommand, service, '127.0.0.1', str(self.port))
+            self.run_networksetup_command(
+                subcommand, service, '127.0.0.1', str(
+                    self.port))
 
     def disable_proxy_for_service(self, service):
         print('Disabling proxy on {}...'.format(service))
@@ -39,14 +45,20 @@ class Wrapper(object):
 
     def interface_name_to_service_name_map(self):
         order = self.run_networksetup_command('-listnetworkserviceorder')
-        mapping = re.findall(r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$', order, re.MULTILINE)
+        mapping = re.findall(
+            r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$',
+            order,
+            re.MULTILINE)
         return dict([(b, a) for (a, b) in mapping])
 
     def run_command_with_input(self, command, input):
-        popen = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        popen = subprocess.Popen(
+            command,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE)
         (stdout, stderr) = popen.communicate(input)
         return stdout
-    
+
     def primary_interace_name(self):
         scutil_script = 'get State:/Network/Global/IPv4\nd.show\n'
         stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
@@ -54,13 +66,15 @@ class Wrapper(object):
         return interface
 
     def primary_service_name(self):
-        return self.interface_name_to_service_name_map()[self.primary_interace_name()]
+        return self.interface_name_to_service_name_map()[
+            self.primary_interace_name()]
 
     def proxy_enabled_for_service(self, service):
         return self.proxy_state_for_service(service)['Enabled'] == 'Yes'
 
     def toggle_proxy(self):
-        new_state = not self.proxy_enabled_for_service(self.primary_service_name())
+        new_state = not self.proxy_enabled_for_service(
+            self.primary_service_name())
         for service_name in self.connected_service_names():
             if self.proxy_enabled_for_service(service_name) and not new_state:
                 self.disable_proxy_for_service(service_name)
@@ -74,8 +88,11 @@ class Wrapper(object):
 
         service_names = []
         for service_id in service_ids:
-            scutil_script = 'show Setup:/Network/Service/{}\n'.format(service_id)
+            scutil_script = 'show Setup:/Network/Service/{}\n'.format(
-            stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
+                service_id)
+            stdout = self.run_command_with_input(
+                '/usr/sbin/scutil',
+                scutil_script)
             service_name, = re.findall(r'UserDefinedName\s*:\s*(.+)', stdout)
             service_names.append(service_name)
 
@@ -102,7 +119,7 @@ class Wrapper(object):
         for service_name in connected_service_names:
             if not self.proxy_enabled_for_service(service_name):
                 self.enable_proxy_for_service(service_name)
-        
+
         yield
 
         for service_name in connected_service_names:
@@ -119,15 +136,23 @@ class Wrapper(object):
     def main(cls):
         parser = argparse.ArgumentParser(
             description='Helper tool for OS X proxy configuration and mitmproxy.',
-            epilog='Any additional arguments will be passed on unchanged to mitmproxy.'
+            epilog='Any additional arguments will be passed on unchanged to mitmproxy.')
-        )
+        parser.add_argument(
-        parser.add_argument('-t', '--toggle', action='store_true', help='just toggle the proxy configuration')
+            '-t',
+            '--toggle',
+            action='store_true',
+            help='just toggle the proxy configuration')
 #         parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy')
-        parser.add_argument('-p', '--port', type=int, help='override the default port of 8080', default=8080)
+        parser.add_argument(
+            '-p',
+            '--port',
+            type=int,
+            help='override the default port of 8080',
+            default=8080)
         args, extra_arguments = parser.parse_known_args()
 
         wrapper = cls(port=args.port, extra_arguments=extra_arguments)
-        
+
         if args.toggle:
             wrapper.toggle_proxy()
 #         elif args.honeyproxy:
@@ -139,4 +164,3 @@ class Wrapper(object):
 if __name__ == '__main__':
     Wrapper.ensure_superuser()
     Wrapper.main()
-

examples/modify_form.py

@@ -1,6 +1,7 @@
 
 def request(context, flow):
-    if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]:
+    if "application/x-www-form-urlencoded" in flow.request.headers[
+            "content-type"]:
         form = flow.request.get_form_urlencoded()
         form["mitmproxy"] = ["rocks"]
-        flow.request.set_form_urlencoded(form)
+        flow.request.set_form_urlencoded(form)

examples/modify_querystring.py

@@ -3,4 +3,4 @@ def request(context, flow):
     q = flow.request.get_query()
     if q:
         q["mitmproxy"] = ["rocks"]
-        flow.request.set_query(q)
+        flow.request.set_query(q)

examples/modify_response_body.py

@@ -6,10 +6,13 @@ from libmproxy.protocol.http import decoded
 def start(context, argv):
     if len(argv) != 3:
         raise ValueError('Usage: -s "modify-response-body.py old new"')
-    # You may want to use Python's argparse for more sophisticated argument parsing.
+    # You may want to use Python's argparse for more sophisticated argument
+    # parsing.
     context.old, context.new = argv[1], argv[2]
 
 
 def response(context, flow):
     with decoded(flow.response):  # automatically decode gzipped responses.
-        flow.response.content = flow.response.content.replace(context.old, context.new)
+        flow.response.content = flow.response.content.replace(
+            context.old,
+            context.new)

examples/proxapp.py

@@ -21,4 +21,4 @@ def start(context, argv):
     # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
     # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
     # but won't send any data.
-    context.app_registry.add(app, "example.com", 443)
+    context.app_registry.add(app, "example.com", 443)

examples/read_dumpfile

@@ -4,7 +4,8 @@
 #
 
 from libmproxy import flow
-import json, sys
+import json
+import sys
 
 with open("logfile", "rb") as logfile:
     freader = flow.FlowReader(logfile)
@@ -14,5 +15,5 @@ with open("logfile", "rb") as logfile:
             print(f.request.host)
             json.dump(f.get_state(), sys.stdout, indent=4)
             print("")
-    except flow.FlowReadError, v:
+    except flow.FlowReadError as v:
-        print("Flow file corrupted. Stopped loading.")
+        print "Flow file corrupted. Stopped loading."

examples/redirect_requests.py

@@ -8,7 +8,8 @@ This example shows two ways to redirect flows to other destinations.
 
 def request(context, flow):
     # pretty_host(hostheader=True) takes the Host: header of the request into account,
-    # which is useful in transparent mode where we usually only have the IP otherwise.
+    # which is useful in transparent mode where we usually only have the IP
+    # otherwise.
 
     # Method 1: Answer with a locally generated response
     if flow.request.pretty_host(hostheader=True).endswith("example.com"):

examples/stream.py

@@ -2,4 +2,4 @@ def responseheaders(context, flow):
     """
     Enables streaming for all responses.
     """
-    flow.response.stream = True
+    flow.response.stream = True

examples/stream_modify.py

@@ -11,7 +11,7 @@ Be aware that content replacement isn't trivial:
 def modify(chunks):
     """
     chunks is a generator that can be used to iterate over all chunks.
-    Each chunk is a (prefix, content, suffix) tuple. 
+    Each chunk is a (prefix, content, suffix) tuple.
     For example, in the case of chunked transfer encoding: ("3\r\n","foo","\r\n")
     """
     for prefix, content, suffix in chunks:
@@ -19,4 +19,4 @@ def modify(chunks):
 
 
 def responseheaders(context, flow):
-    flow.response.stream = modify
+    flow.response.stream = modify

examples/stub.py

@@ -1,12 +1,15 @@
 """
     This is a script stub, with definitions for all events.
 """
+
+
 def start(context, argv):
     """
         Called once on script startup, before any other events.
     """
     context.log("start")
 
+
 def clientconnect(context, conn_handler):
     """
         Called when a client initiates a connection to the proxy. Note that a
@@ -14,6 +17,7 @@ def clientconnect(context, conn_handler):
     """
     context.log("clientconnect")
 
+
 def serverconnect(context, conn_handler):
     """
         Called when the proxy initiates a connection to the target server. Note that a
@@ -21,6 +25,7 @@ def serverconnect(context, conn_handler):
     """
     context.log("serverconnect")
 
+
 def request(context, flow):
     """
         Called when a client request has been received.
@@ -36,12 +41,14 @@ def responseheaders(context, flow):
     """
     context.log("responseheaders")
 
+
 def response(context, flow):
     """
        Called when a server response has been received.
     """
     context.log("response")
 
+
 def error(context, flow):
     """
         Called when a flow error has occured, e.g. invalid server responses, or
@@ -50,12 +57,14 @@ def error(context, flow):
     """
     context.log("error")
 
+
 def clientdisconnect(context, conn_handler):
     """
         Called when a client disconnects from the proxy.
     """
     context.log("clientdisconnect")
 
+
 def done(context):
     """
         Called once on script shutdown, after any other events.

examples/upsidedownternet.py

@@ -2,6 +2,7 @@ import cStringIO
 from PIL import Image
 from libmproxy.protocol.http import decoded
 
+
 def response(context, flow):
     if flow.response.headers.get_first("content-type", "").startswith("image"):
         with decoded(flow.response):  # automatically decode gzipped responses.
@@ -13,4 +14,4 @@ def response(context, flow):
                 flow.response.content = s2.getvalue()
                 flow.response.headers["content-type"] = ["image/png"]
             except:  # Unknown image types etc.
-                pass
+                pass

libmproxy/cmdline.py

@@ -65,7 +65,7 @@ def parse_replace_hook(s):
     patt, regex, replacement = _parse_hook(s)
     try:
         re.compile(regex)
-    except re.error, e:
+    except re.error as e:
         raise ParseException("Malformed replacement regex: %s" % str(e.message))
     return patt, regex, replacement
 
@@ -127,7 +127,6 @@ def parse_server_spec_special(url):
     return ret
 
 
-
 def get_common_options(options):
     stickycookie, stickyauth = None, None
     if options.stickycookie_filt:
@@ -142,17 +141,17 @@ def get_common_options(options):
     for i in options.replace:
         try:
             p = parse_replace_hook(i)
-        except ParseException, e:
+        except ParseException as e:
             raise configargparse.ArgumentTypeError(e.message)
         reps.append(p)
     for i in options.replace_file:
         try:
             patt, rex, path = parse_replace_hook(i)
-        except ParseException, e:
+        except ParseException as e:
             raise configargparse.ArgumentTypeError(e.message)
         try:
             v = open(path, "rb").read()
-        except IOError, e:
+        except IOError as e:
             raise configargparse.ArgumentTypeError(
                 "Could not read replace file: %s" % path
             )
@@ -162,7 +161,7 @@ def get_common_options(options):
     for i in options.setheader:
         try:
             p = parse_setheader(i)
-        except ParseException, e:
+        except ParseException as e:
             raise configargparse.ArgumentTypeError(e.message)
         setheaders.append(p)
 
@@ -221,7 +220,7 @@ def common_options(parser):
     parser.add_argument(
         "--cadir",
         action="store", type=str, dest="cadir", default=config.CA_DIR,
-        help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR
+        help="Location of the default mitmproxy CA files. (%s)" % config.CA_DIR
     )
     parser.add_argument(
         "--host",
@@ -466,7 +465,7 @@ def common_options(parser):
         "--replay-ignore-payload-param",
         action="append", dest="replay_ignore_payload_params", type=str,
         help="""
-            Request's payload parameters (application/x-www-form-urlencoded) to
+            Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to
             be ignored while searching for a saved flow to replay.
             Can be passed multiple times.
         """
@@ -482,9 +481,10 @@ def common_options(parser):
     )
     group.add_argument(
         "--replay-ignore-host",
-        action="store_true", dest="replay_ignore_host", default=False,
+        action="store_true",
-        help="Ignore request's destination host while searching for a saved flow to replay"
+        dest="replay_ignore_host",
-    )
+        default=False,
+        help="Ignore request's destination host while searching for a saved flow to replay")
 
     group = parser.add_argument_group(
         "Replacements",
@@ -575,11 +575,16 @@ def mitmproxy():
     )
     common_options(parser)
     parser.add_argument(
-        "--palette", type=str, default="dark",
+        "--palette", type=str, default=palettes.DEFAULT,
         action="store", dest="palette",
         choices=sorted(palettes.palettes.keys()),
         help="Select color palette: " + ", ".join(palettes.palettes.keys())
     )
+    parser.add_argument(
+        "--palette-transparent",
+        action="store_true", dest="palette_transparent", default=False,
+        help="Set transparent background for palette."
+    )
     parser.add_argument(
         "-e", "--eventlog",
         action="store_true", dest="eventlog",
@@ -594,6 +599,11 @@ def mitmproxy():
         type=str, dest="intercept", default=None,
         help="Intercept filter expression."
     )
+    group.add_argument(
+        "-l", "--limit", action="store",
+        type=str, dest="limit", default=None,
+        help="Limit filter expression."
+    )
     return parser
 
 

libmproxy/console/common.py

@@ -6,15 +6,14 @@ import os
 
 from .. import utils
 from ..protocol.http import CONTENT_MISSING, decoded
+from . import signals
+import netlib.utils
 
 try:
     import pyperclip
 except:
     pyperclip = False
 
-VIEW_LIST = 0
-VIEW_FLOW = 1
-
 
 VIEW_FLOW_REQUEST = 0
 VIEW_FLOW_RESPONSE = 1
@@ -31,14 +30,22 @@ METHOD_OPTIONS = [
 ]
 
 
-def highlight_key(s, k):
+def is_keypress(k):
+    """
+        Is this input event a keypress?
+    """
+    if isinstance(k, basestring):
+        return True
+
+
+def highlight_key(str, key, textattr="text", keyattr="key"):
     l = []
-    parts = s.split(k, 1)
+    parts = str.split(key, 1)
     if parts[0]:
-        l.append(("text", parts[0]))
+        l.append((textattr, parts[0]))
-    l.append(("key", k))
+    l.append((keyattr, key))
     if parts[1]:
-        l.append(("text", parts[1]))
+        l.append((textattr, parts[1]))
     return l
 
 
@@ -60,20 +67,26 @@ def format_keyvals(lst, key="key", val="text", indent=0):
             if kv is None:
                 ret.append(urwid.Text(""))
             else:
-                cols = []
+                if isinstance(kv[1], urwid.Widget):
-                # This cumbersome construction process is here for a reason:
+                    v = kv[1]
-                # Urwid < 1.0 barfs if given a fixed size column of size zero.
+                elif kv[1] is None:
-                if indent:
+                    v = urwid.Text("")
-                    cols.append(("fixed", indent, urwid.Text("")))
+                else:
-                cols.extend([
+                    v = urwid.Text([(val, kv[1])])
-                    (
+                ret.append(
-                        "fixed",
+                    urwid.Columns(
-                        maxk,
+                        [
-                        urwid.Text([(key, kv[0] or "")])
+                            ("fixed", indent, urwid.Text("")),
-                    ),
+                            (
-                    kv[1] if isinstance(kv[1], urwid.Widget) else urwid.Text([(val, kv[1])])
+                                "fixed",
-               ])
+                                maxk,
-                ret.append(urwid.Columns(cols, dividechars = 2))
+                                urwid.Text([(key, kv[0] or "")])
+                            ),
+                            v
+                        ],
+                        dividechars = 2
+                    )
+                )
     return ret
 
 
@@ -151,7 +164,7 @@ def raw_format_flow(f, focus, extended, padding):
             4: "code_400",
             5: "code_500",
         }
-        ccol = codes.get(f["resp_code"]/100, "code_other")
+        ccol = codes.get(f["resp_code"] / 100, "code_other")
         resp.append(fcol(SYMBOL_RETURN, ccol))
         if f["resp_is_replay"]:
             resp.append(fcol(SYMBOL_REPLAY, "replay"))
@@ -184,23 +197,39 @@ def raw_format_flow(f, focus, extended, padding):
 def save_data(path, data, master, state):
     if not path:
         return
-    state.last_saveload = path
-    path = os.path.expanduser(path)
     try:
         with file(path, "wb") as f:
             f.write(data)
-    except IOError, v:
+    except IOError as v:
-        master.statusbar.message(v.strerror)
+        signals.status_message.send(message=v.strerror)
+
+
+def ask_save_overwite(path, data, master, state):
+    if not path:
+        return
+    path = os.path.expanduser(path)
+    if os.path.exists(path):
+        def save_overwite(k):
+            if k == "y":
+                save_data(path, data, master, state)
+
+        signals.status_prompt_onekey.send(
+            prompt = "'" + path + "' already exists. Overwite?",
+            keys = (
+                ("yes", "y"),
+                ("no", "n"),
+            ),
+            callback = save_overwite
+        )
+    else:
+        save_data(path, data, master, state)
 
 
 def ask_save_path(prompt, data, master, state):
-    master.path_prompt(
+    signals.status_prompt_path.send(
-        prompt,
+        prompt = prompt,
-        state.last_saveload,
+        callback = ask_save_overwite,
-        save_data,
+        args = (data, master, state)
-        data,
-        master,
-        state
     )
 
 
@@ -210,6 +239,8 @@ def copy_flow_format_data(part, scope, flow):
     else:
         data = ""
         if scope in ("q", "a"):
+            if flow.request.content is None or flow.request.content == CONTENT_MISSING:
+                return None, "Request content is missing"
             with decoded(flow.request):
                 if part == "h":
                     data += flow.request.assemble()
@@ -221,6 +252,8 @@ def copy_flow_format_data(part, scope, flow):
             # Add padding between request and response
             data += "\r\n" * 2
         if scope in ("s", "a") and flow.response:
+            if flow.response.content is None or flow.response.content == CONTENT_MISSING:
+                return None, "Response content is missing"
             with decoded(flow.response):
                 if part == "h":
                     data += flow.response.assemble()
@@ -228,40 +261,43 @@ def copy_flow_format_data(part, scope, flow):
                     data += flow.response.content
                 else:
                     raise ValueError("Unknown part: {}".format(part))
-    return data
+    return data, False
 
 
 def copy_flow(part, scope, flow, master, state):
     """
-    part: _c_ontent, _a_ll, _u_rl
+    part: _c_ontent, _h_eaders+content, _u_rl
     scope: _a_ll, re_q_uest, re_s_ponse
     """
-    data = copy_flow_format_data(part, scope, flow)
+    data, err = copy_flow_format_data(part, scope, flow)
+
+    if err:
+        signals.status_message.send(message=err)
+        return
 
     if not data:
         if scope == "q":
-            master.statusbar.message("No request content to copy.")
+            signals.status_message.send(message="No request content to copy.")
         elif scope == "s":
-            master.statusbar.message("No response content to copy.")
+            signals.status_message.send(message="No response content to copy.")
         else:
-            master.statusbar.message("No contents to copy.")
+            signals.status_message.send(message="No contents to copy.")
         return
 
     try:
         master.add_event(str(len(data)))
         pyperclip.copy(data)
-    except RuntimeError:
+    except (RuntimeError, UnicodeDecodeError):
         def save(k):
             if k == "y":
-                ask_save_path("Save data: ", data, master, state)
+                ask_save_path("Save data", data, master, state)
-
+        signals.status_prompt_onekey.send(
-        master.prompt_onekey(
+            prompt = "Cannot copy binary data to clipboard. Save as file?",
-            "Cannot copy binary data to clipboard. Save as file?",
+            keys = (
-            (
                 ("yes", "y"),
                 ("no", "n"),
             ),
-            save
+            callback = save
         )
 
 
@@ -273,14 +309,11 @@ def ask_copy_part(scope, flow, master, state):
     if scope != "s":
         choices.append(("url", "u"))
 
-    master.prompt_onekey(
+    signals.status_prompt_onekey.send(
-        "Copy",
+        prompt = "Copy",
-        choices,
+        keys = choices,
-        copy_flow,
+        callback = copy_flow,
-        scope,
+        args = (scope, flow, master, state)
-        flow,
-        master,
-        state
     )
 
 
@@ -297,16 +330,14 @@ def ask_save_body(part, master, state, flow):
         # We first need to determine whether we want to save the request or the
         # response content.
         if request_has_content and response_has_content:
-            master.prompt_onekey(
+            signals.status_prompt_onekey.send(
-                "Save",
+                prompt = "Save",
-                (
+                keys = (
                     ("request", "q"),
                     ("response", "s"),
                 ),
-                ask_save_body,
+                callback = ask_save_body,
-                master,
+                args = (master, state, flow)
-                state,
-                flow
             )
         elif response_has_content:
             ask_save_body("s", master, state, flow)
@@ -315,27 +346,23 @@ def ask_save_body(part, master, state, flow):
 
     elif part == "q" and request_has_content:
         ask_save_path(
-            "Save request content: ",
+            "Save request content",
             flow.request.get_decoded_content(),
             master,
             state
         )
     elif part == "s" and response_has_content:
         ask_save_path(
-            "Save response content: ",
+            "Save response content",
             flow.response.get_decoded_content(),
             master,
             state
         )
     else:
-        master.statusbar.message("No content to save.")
+        signals.status_message.send(message="No content to save.")
 
 
-class FlowCache:
+flowcache = utils.LRUCache(800)
-    @utils.LRUCache(200)
-    def format_flow(self, *args):
-        return raw_format_flow(*args)
-flowcache = FlowCache()
 
 
 def format_flow(f, focus, extended=False, hostheader=False, padding=2):
@@ -353,7 +380,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
     )
     if f.response:
         if f.response.content:
-            contentdesc = utils.pretty_size(len(f.response.content))
+            contentdesc = netlib.utils.pretty_size(len(f.response.content))
         elif f.response.content == CONTENT_MISSING:
             contentdesc = "[content missing]"
         else:
@@ -374,6 +401,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
             d["resp_ctype"] = t[0].split(";")[0]
         else:
             d["resp_ctype"] = ""
-    return flowcache.format_flow(
+    return flowcache.get(
+        raw_format_flow,
         tuple(sorted(d.items())), focus, extended, padding
     )

libmproxy/console/contentview.py

@@ -6,27 +6,27 @@ import lxml.html
 import lxml.etree
 from PIL import Image
 from PIL.ExifTags import TAGS
-import re
 import subprocess
 import traceback
 import urwid
 
 import netlib.utils
+from netlib import odict
 
 from . import common
-from .. import utils, encoding, flow
+from .. import utils, encoding
 from ..contrib import jsbeautifier, html2text
 from ..contrib.wbxml.ASCommandResponse import ASCommandResponse
 
 try:
     import pyamf
     from pyamf import remoting, flex
-except ImportError: # pragma nocover
+except ImportError:  # pragma nocover
     pyamf = None
 
 try:
     import cssutils
-except ImportError: # pragma nocover
+except ImportError:  # pragma nocover
     cssutils = None
 else:
     cssutils.log.setLevel(logging.CRITICAL)
@@ -36,7 +36,7 @@ else:
     cssutils.ser.prefs.indentClosingBrace = False
     cssutils.ser.prefs.validOnly = False
 
-VIEW_CUTOFF = 1024*50
+VIEW_CUTOFF = 1024 * 50
 
 
 def _view_text(content, total, limit):
@@ -59,7 +59,7 @@ def trailer(clen, txt, limit):
         txt.append(
             urwid.Text(
                 [
-                    ("highlight", "... %s of data not shown. Press "%utils.pretty_size(rem)),
+                    ("highlight", "... %s of data not shown. Press " % netlib.utils.pretty_size(rem)),
                     ("key", "f"),
                     ("highlight", " to load all data.")
                 ]
@@ -76,7 +76,7 @@ class ViewAuto:
         ctype = hdrs.get_first("content-type")
         if ctype:
             ct = utils.parse_content_type(ctype) if ctype else None
-            ct = "%s/%s"%(ct[0], ct[1])
+            ct = "%s/%s" % (ct[0], ct[1])
             if ct in content_types_map:
                 return content_types_map[ct][0](hdrs, content, limit)
             elif utils.isXML(content):
@@ -227,7 +227,7 @@ class ViewURLEncoded:
         lines = utils.urldecode(content)
         if lines:
             body = common.format_keyvals(
-                [(k+":", v) for (k, v) in lines],
+                [(k + ":", v) for (k, v) in lines],
                 key = "header",
                 val = "text"
             )
@@ -240,33 +240,13 @@ class ViewMultipart:
     content_types = ["multipart/form-data"]
 
     def __call__(self, hdrs, content, limit):
-        v = hdrs.get_first("content-type")
+        v = utils.multipartdecode(hdrs, content)
         if v:
-            v = utils.parse_content_type(v)
-            if not v:
-                return
-            boundary = v[2].get("boundary")
-            if not boundary:
-                return
-
-            rx = re.compile(r'\bname="([^"]+)"')
-            keys = []
-            vals = []
-
-            for i in content.split("--" + boundary):
-                parts = i.splitlines()
-                if len(parts) > 1 and parts[0][0:2] != "--":
-                    match = rx.search(parts[1])
-                    if match:
-                        keys.append(match.group(1) + ":")
-                        vals.append(netlib.utils.cleanBin(
-                            "\n".join(parts[3+parts[2:].index(""):])
-                        ))
             r = [
                 urwid.Text(("highlight", "Form data:\n")),
             ]
             r.extend(common.format_keyvals(
-                zip(keys, vals),
+                v,
                 key = "header",
                 val = "text"
             ))
@@ -324,7 +304,6 @@ if pyamf:
             if not envelope:
                 return None
 
-
             txt = []
             for target, message in iter(envelope):
                 if isinstance(message, pyamf.remoting.Request):
@@ -335,13 +314,13 @@ if pyamf:
                 else:
                     txt.append(urwid.Text([
                         ("header", "Response: "),
-                        ("text", "%s, code %s"%(target, message.status)),
+                        ("text", "%s, code %s" % (target, message.status)),
                     ]))
 
                 s = json.dumps(self.unpack(message), indent=4)
                 txt.extend(_view_text(s[:limit], len(s), limit))
 
-            return "AMF v%s"%envelope.amfVersion, txt
+            return "AMF v%s" % envelope.amfVersion, txt
 
 
 class ViewJavaScript:
@@ -395,7 +374,7 @@ class ViewImage:
             return None
         parts = [
             ("Format", str(img.format_description)),
-            ("Size", "%s x %s px"%img.size),
+            ("Size", "%s x %s px" % img.size),
             ("Mode", str(img.mode)),
         ]
         for i in sorted(img.info.keys()):
@@ -421,7 +400,7 @@ class ViewImage:
             key = "header",
             val = "text"
         )
-        return "%s image"%img.format, fmt
+        return "%s image" % img.format, fmt
 
 
 class ViewProtobuf:
@@ -539,14 +518,14 @@ def get_content_view(viewmode, hdrItems, content, limit, logfunc, is_request):
             return "No content", ""
     msg = []
 
-    hdrs = flow.ODictCaseless([list(i) for i in hdrItems])
+    hdrs = odict.ODictCaseless([list(i) for i in hdrItems])
 
     enc = hdrs.get_first("content-encoding")
     if enc and enc != "identity":
         decoded = encoding.decode(enc, content)
         if decoded:
             content = decoded
-            msg.append("[decoded %s]"%enc)
+            msg.append("[decoded %s]" % enc)
     try:
         ret = viewmode(hdrs, content, limit)
     # Third-party viewers can fail in unexpected ways...

libmproxy/console/flowdetailview.py

@@ -1,113 +1,154 @@
 from __future__ import absolute_import
 import urwid
-from . import common
+from . import common, searchable
 from .. import utils
 
-footer = [
-    ('heading_key', "q"), ":back ",
-]
 
-class FlowDetailsView(urwid.ListBox):
+def maybe_timestamp(base, attr):
-    def __init__(self, master, flow, state):
+    if base and getattr(base, attr):
-        self.master, self.flow, self.state = master, flow, state
+        return utils.format_timestamp_with_milli(getattr(base, attr))
-        urwid.ListBox.__init__(
+    else:
-            self,
+        return "active"
-            self.flowtext()
+    pass
+
+
+def flowdetails(state, flow):
+    text = []
+
+    cc = flow.client_conn
+    sc = flow.server_conn
+    req = flow.request
+    resp = flow.response
+
+    if sc:
+        text.append(urwid.Text([("head", "Server Connection:")]))
+        parts = [
+            ["Address", "%s:%s" % sc.address()],
+        ]
+
+        text.extend(
+            common.format_keyvals(parts, key="key", val="text", indent=4)
         )
 
-    def keypress(self, size, key):
+        c = sc.cert
-        key = common.shortcuts(key)
+        if c:
-        if key == "q":
+            text.append(urwid.Text([("head", "Server Certificate:")]))
-            self.master.statusbar = self.state[0]
-            self.master.body = self.state[1]
-            self.master.header = self.state[2]
-            self.master.loop.widget = self.master.make_view()
-            return None
-        elif key == "?":
-            key = None
-        return urwid.ListBox.keypress(self, size, key)
-
-    def flowtext(self):
-        text = []
-
-        title = urwid.Text("Flow details")
-        title = urwid.Padding(title, align="left", width=("relative", 100))
-        title = urwid.AttrWrap(title, "heading")
-        text.append(title)
-
-        cc = self.flow.client_conn
-        sc = self.flow.server_conn
-        req = self.flow.request
-        resp = self.flow.response
-
-        if sc:
-            text.append(urwid.Text([("head", "Server Connection:")]))
             parts = [
-                ["Address", "%s:%s" % sc.address()],
+                ["Type", "%s, %s bits" % c.keyinfo],
-            ]
+                ["SHA1 digest", c.digest("sha1")],
-
+                ["Valid to", str(c.notafter)],
-            text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+                ["Valid from", str(c.notbefore)],
-
+                ["Serial", str(c.serial)],
-            c = sc.cert
+                [
-            if c:
+                    "Subject",
-                text.append(urwid.Text([("head", "Server Certificate:")]))
+                    urwid.BoxAdapter(
-                parts = [
+                        urwid.ListBox(
-                    ["Type", "%s, %s bits"%c.keyinfo],
+                            common.format_keyvals(
-                    ["SHA1 digest", c.digest("sha1")],
+                                c.subject,
-                    ["Valid to", str(c.notafter)],
+                                key="highlight",
-                    ["Valid from", str(c.notbefore)],
+                                val="text"
-                    ["Serial", str(c.serial)],
+                            )
-                    [
+                        ),
-                        "Subject",
+                        len(c.subject)
-                        urwid.BoxAdapter(
-                            urwid.ListBox(common.format_keyvals(c.subject, key="highlight", val="text")),
-                            len(c.subject)
-                        )
-                    ],
-                    [
-                        "Issuer",
-                        urwid.BoxAdapter(
-                            urwid.ListBox(common.format_keyvals(c.issuer, key="highlight", val="text")),
-                            len(c.issuer)
-                        )
-                    ]
-                ]
-
-                if c.altnames:
-                    parts.append(
-                        [
-                            "Alt names",
-                            ", ".join(c.altnames)
-                        ]
                     )
-                text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+                ],
-
+                [
-        if cc:
+                    "Issuer",
-            text.append(urwid.Text([("head", "Client Connection:")]))
+                    urwid.BoxAdapter(
-
+                        urwid.ListBox(
-            parts = [
+                            common.format_keyvals(
-                ["Address", "%s:%s" % cc.address()],
+                                c.issuer, key="highlight", val="text"
-                # ["Requests", "%s"%cc.requestcount],
+                            )
+                        ),
+                        len(c.issuer)
+                    )
+                ]
             ]
 
-            text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+            if c.altnames:
+                parts.append(
+                    [
+                        "Alt names",
+                        ", ".join(c.altnames)
+                    ]
+                )
+            text.extend(
+                common.format_keyvals(parts, key="key", val="text", indent=4)
+            )
 
-        parts = []
+    if cc:
+        text.append(urwid.Text([("head", "Client Connection:")]))
 
-        parts.append(["Client conn. established", utils.format_timestamp_with_milli(cc.timestamp_start) if (cc and cc.timestamp_start) else "active"])
+        parts = [
-        parts.append(["Server conn. initiated", utils.format_timestamp_with_milli(sc.timestamp_start) if sc else "active" ])
+            ["Address", "%s:%s" % cc.address()],
-        parts.append(["Server conn. TCP handshake", utils.format_timestamp_with_milli(sc.timestamp_tcp_setup) if (sc and sc.timestamp_tcp_setup) else "active"])
+            # ["Requests", "%s"%cc.requestcount],
-        if sc.ssl_established:
+        ]
-            parts.append(["Server conn. SSL handshake", utils.format_timestamp_with_milli(sc.timestamp_ssl_setup) if sc.timestamp_ssl_setup else "active"])
-            parts.append(["Client conn. SSL handshake", utils.format_timestamp_with_milli(cc.timestamp_ssl_setup) if (cc and cc.timestamp_ssl_setup) else "active"])
-        parts.append(["First request byte", utils.format_timestamp_with_milli(req.timestamp_start)])
-        parts.append(["Request complete", utils.format_timestamp_with_milli(req.timestamp_end) if req.timestamp_end else "active"])
-        parts.append(["First response byte", utils.format_timestamp_with_milli(resp.timestamp_start) if resp else "active"])
-        parts.append(["Response complete", utils.format_timestamp_with_milli(resp.timestamp_end) if (resp and resp.timestamp_end) else "active"])
 
-        # sort operations by timestamp
+        text.extend(
-        parts = sorted(parts, key=lambda p: p[1])
+            common.format_keyvals(parts, key="key", val="text", indent=4)
+        )
 
-        text.append(urwid.Text([("head", "Timing:")]))
+    parts = []
-        text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+
-        return text
+    parts.append(
+        [
+            "Client conn. established",
+            maybe_timestamp(cc, "timestamp_start")
+        ]
+    )
+    parts.append(
+        [
+            "Server conn. initiated",
+            maybe_timestamp(sc, "timestamp_start")
+        ]
+    )
+    parts.append(
+        [
+            "Server conn. TCP handshake",
+            maybe_timestamp(sc, "timestamp_tcp_setup")
+        ]
+    )
+    if sc.ssl_established:
+        parts.append(
+            [
+                "Server conn. SSL handshake",
+                maybe_timestamp(sc, "timestamp_ssl_setup")
+            ]
+        )
+        parts.append(
+            [
+                "Client conn. SSL handshake",
+                maybe_timestamp(cc, "timestamp_ssl_setup")
+            ]
+        )
+    parts.append(
+        [
+            "First request byte",
+            maybe_timestamp(req, "timestamp_start")
+        ]
+    )
+    parts.append(
+        [
+            "Request complete",
+            maybe_timestamp(req, "timestamp_end")
+        ]
+    )
+    parts.append(
+        [
+            "First response byte",
+            maybe_timestamp(resp, "timestamp_start")
+        ]
+    )
+    parts.append(
+        [
+            "Response complete",
+            maybe_timestamp(resp, "timestamp_end")
+        ]
+    )
+
+    # sort operations by timestamp
+    parts = sorted(parts, key=lambda p: p[1])
+
+    text.append(urwid.Text([("head", "Timing:")]))
+    text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
+    return searchable.Searchable(state, text)

libmproxy/console/flowlist.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 import urwid
 from netlib import http
-from . import common
+from . import common, signals
 
 
 def _mkhelp():
@@ -15,10 +15,10 @@ def _mkhelp():
         ("D", "duplicate flow"),
         ("e", "toggle eventlog"),
         ("F", "toggle follow flow list"),
-        ("g", "copy flow to clipboard"),
         ("l", "set limit filter pattern"),
         ("L", "load saved flows"),
         ("n", "create a new request"),
+        ("P", "copy flow to clipboard"),
         ("r", "replay request"),
         ("V", "revert changes to request"),
         ("w", "save flows "),
@@ -47,6 +47,10 @@ class EventListBox(urwid.ListBox):
         if key == "C":
             self.master.clear_events()
             key = None
+        elif key == "G":
+            self.set_focus(0)
+        elif key == "g":
+            self.set_focus(len(self.master.eventlist) - 1)
         return urwid.ListBox.keypress(self, size, key)
 
 
@@ -72,7 +76,8 @@ class BodyPile(urwid.Pile):
 
     def keypress(self, size, key):
         if key == "tab":
-            self.focus_position = (self.focus_position + 1)%len(self.widget_list)
+            self.focus_position = (
+                self.focus_position + 1) % len(self.widget_list)
             if self.focus_position == 1:
                 self.widget_list[1].header = self.active_header
             else:
@@ -111,17 +116,15 @@ class ConnectionItem(urwid.WidgetWrap):
 
     def save_flows_prompt(self, k):
         if k == "a":
-            self.master.path_prompt(
+            signals.status_prompt_path.send(
-                "Save all flows to: ",
+                prompt = "Save all flows to",
-                self.state.last_saveload,
+                callback = self.master.save_flows
-                self.master.save_flows
             )
         else:
-            self.master.path_prompt(
+            signals.status_prompt_path.send(
-                "Save this flow to: ",
+                prompt = "Save this flow to",
-                self.state.last_saveload,
+                callback = self.master.save_one_flow,
-                self.master.save_one_flow,
+                args = (self.flow,)
-                self.flow
             )
 
     def stop_server_playback_prompt(self, a):
@@ -150,64 +153,65 @@ class ConnectionItem(urwid.WidgetWrap):
                 self.master.options.replay_ignore_host
             )
         else:
-            self.master.path_prompt(
+            signals.status_prompt_path.send(
-                "Server replay path: ",
+                prompt = "Server replay path",
-                self.state.last_saveload,
+                callback = self.master.server_playback_path
-                self.master.server_playback_path
             )
 
-    def keypress(self, (maxcol,), key):
+    def keypress(self, xxx_todo_changeme, key):
+        (maxcol,) = xxx_todo_changeme
         key = common.shortcuts(key)
         if key == "a":
             self.flow.accept_intercept(self.master)
-            self.master.sync_list_view()
+            signals.flowlist_change.send(self)
         elif key == "d":
             self.flow.kill(self.master)
             self.state.delete_flow(self.flow)
-            self.master.sync_list_view()
+            signals.flowlist_change.send(self)
         elif key == "D":
             f = self.master.duplicate_flow(self.flow)
             self.master.view_flow(f)
         elif key == "r":
             r = self.master.replay_request(self.flow)
             if r:
-                self.master.statusbar.message(r)
+                signals.status_message.send(message=r)
-            self.master.sync_list_view()
+            signals.flowlist_change.send(self)
         elif key == "S":
             if not self.master.server_playback:
-                self.master.prompt_onekey(
+                signals.status_prompt_onekey.send(
-                    "Server Replay",
+                    prompt = "Server Replay",
-                    (
+                    keys = (
                         ("all flows", "a"),
                         ("this flow", "t"),
                         ("file", "f"),
                     ),
-                    self.server_replay_prompt,
+                    callback = self.server_replay_prompt,
                 )
             else:
-                self.master.prompt_onekey(
+                signals.status_prompt_onekey.send(
-                    "Stop current server replay?",
+                    prompt = "Stop current server replay?",
-                    (
+                    keys = (
                         ("yes", "y"),
                         ("no", "n"),
                     ),
-                    self.stop_server_playback_prompt,
+                    callback = self.stop_server_playback_prompt,
                 )
         elif key == "V":
             if not self.flow.modified():
-                self.master.statusbar.message("Flow not modified.")
+                signals.status_message.send(message="Flow not modified.")
                 return
             self.state.revert(self.flow)
-            self.master.sync_list_view()
+            signals.flowlist_change.send(self)
-            self.master.statusbar.message("Reverted.")
+            signals.status_message.send(message="Reverted.")
         elif key == "w":
-            self.master.prompt_onekey(
+            signals.status_prompt_onekey.send(
-                "Save",
+                self,
-                (
+                prompt = "Save",
+                keys = (
                     ("all flows", "a"),
                     ("this flow", "t"),
                 ),
-                self.save_flows_prompt,
+                callback = self.save_flows_prompt,
             )
         elif key == "X":
             self.flow.kill(self.master)
@@ -215,13 +219,12 @@ class ConnectionItem(urwid.WidgetWrap):
             if self.flow.request:
                 self.master.view_flow(self.flow)
         elif key == "|":
-            self.master.path_prompt(
+            signals.status_prompt_path.send(
-                "Send flow to script: ",
+                prompt = "Send flow to script",
-                self.state.last_script,
+                callback = self.master.run_script_once,
-                self.master.run_script_once,
+                args = (self.flow,)
-                self.flow
             )
-        elif key == "g":
+        elif key == "P":
             common.ask_copy_part("a", self.flow, self.master, self.state)
         elif key == "b":
             common.ask_save_body(None, self.master, self.state, self.flow)
@@ -232,8 +235,10 @@ class ConnectionItem(urwid.WidgetWrap):
 class FlowListWalker(urwid.ListWalker):
     def __init__(self, master, state):
         self.master, self.state = master, state
-        if self.state.flow_count():
+        signals.flowlist_change.connect(self.sig_flowlist_change)
-            self.set_focus(0)
+
+    def sig_flowlist_change(self, sender):
+        self._modified()
 
     def get_focus(self):
         f, i = self.state.get_focus()
@@ -258,7 +263,10 @@ class FlowListWalker(urwid.ListWalker):
 class FlowListBox(urwid.ListBox):
     def __init__(self, master):
         self.master = master
-        urwid.ListBox.__init__(self, master.flow_list_walker)
+        urwid.ListBox.__init__(
+            self,
+            FlowListWalker(master, master.state)
+        )
 
     def get_method_raw(self, k):
         if k:
@@ -266,7 +274,12 @@ class FlowListBox(urwid.ListBox):
 
     def get_method(self, k):
         if k == "e":
-            self.master.prompt("Method:", "", self.get_method_raw)
+            signals.status_prompt.send(
+                self,
+                prompt = "Method",
+                text = "",
+                callback = self.get_method_raw
+            )
         else:
             method = ""
             for i in common.METHOD_OPTIONS:
@@ -275,17 +288,17 @@ class FlowListBox(urwid.ListBox):
             self.get_url(method)
 
     def get_url(self, method):
-        self.master.prompt(
+        signals.status_prompt.send(
-            "URL:",
+            prompt = "URL",
-            "http://www.example.com/",
+            text = "http://www.example.com/",
-            self.new_request,
+            callback = self.new_request,
-            method
+            args = (method,)
         )
 
     def new_request(self, url, method):
         parts = http.parse_url(str(url))
         if not parts:
-            self.master.statusbar.message("Invalid Url")
+            signals.status_message.send(message="Invalid Url")
             return
         scheme, host, port, path = parts
         f = self.master.create_request(method, scheme, host, port, path)
@@ -295,28 +308,34 @@ class FlowListBox(urwid.ListBox):
         key = common.shortcuts(key)
         if key == "A":
             self.master.accept_all()
-            self.master.sync_list_view()
+            signals.flowlist_change.send(self)
         elif key == "C":
             self.master.clear_flows()
         elif key == "e":
             self.master.toggle_eventlog()
+        elif key == "G":
+            self.master.state.set_focus(0)
+            signals.flowlist_change.send(self)
+        elif key == "g":
+            self.master.state.set_focus(self.master.state.flow_count())
+            signals.flowlist_change.send(self)
         elif key == "l":
-            self.master.prompt(
+            signals.status_prompt.send(
-                "Limit: ",
+                prompt = "Limit",
-                self.master.state.limit_txt,
+                text = self.master.state.limit_txt,
-                self.master.set_limit
+                callback = self.master.set_limit
             )
         elif key == "L":
-            self.master.path_prompt(
+            signals.status_prompt_path.send(
-                "Load flows: ",
+                self,
-                self.master.state.last_saveload,
+                prompt = "Load flows",
-                self.master.load_flows_callback
+                callback = self.master.load_flows_callback
             )
         elif key == "n":
-            self.master.prompt_onekey(
+            signals.status_prompt_onekey.send(
-                "Method",
+                prompt = "Method",
-                common.METHOD_OPTIONS,
+                keys = common.METHOD_OPTIONS,
-                self.get_method
+                callback = self.get_method
             )
         elif key == "F":
             self.master.toggle_follow_flows()
@@ -324,10 +343,10 @@ class FlowListBox(urwid.ListBox):
             if self.master.stream:
                 self.master.stop_stream()
             else:
-                self.master.path_prompt(
+                signals.status_prompt_path.send(
-                    "Stream flows to: ",
+                    self,
-                    self.master.state.last_saveload,
+                    prompt = "Stream flows to",
-                    self.master.start_stream_to_path
+                    callback = self.master.start_stream_to_path
                 )
         else:
             return urwid.ListBox.keypress(self, size, key)

libmproxy/console/grideditor.py

@@ -5,31 +5,99 @@ import re
 import os
 import urwid
 
-from . import common
+from . import common, signals
 from .. import utils, filt, script
-from netlib import http_uastrings
+from netlib import http_uastrings, http_cookies, odict
 
 
-footer = [
+FOOTER = [
     ('heading_key', "enter"), ":edit ",
     ('heading_key', "q"), ":back ",
 ]
-footer_editing = [
+FOOTER_EDITING = [
     ('heading_key', "esc"), ":stop editing ",
 ]
 
 
-class SText(urwid.WidgetWrap):
+class TextColumn:
-    def __init__(self, txt, focused, error):
+    subeditor = None
+
+    def __init__(self, heading):
+        self.heading = heading
+
+    def text(self, obj):
+        return SEscaped(obj or "")
+
+    def blank(self):
+        return ""
+
+    def keypress(self, key, editor):
+        if key == "r":
+            if editor.walker.get_current_value() is not None:
+                signals.status_prompt_path.send(
+                    self,
+                    prompt = "Read file",
+                    callback = editor.read_file
+                )
+        elif key == "R":
+            if editor.walker.get_current_value() is not None:
+                signals.status_prompt_path.send(
+                    editor,
+                    prompt = "Read unescaped file",
+                    callback = editor.read_file,
+                    args = (True,)
+                )
+        elif key == "e":
+            o = editor.walker.get_current_value()
+            if o is not None:
+                n = editor.master.spawn_editor(o.encode("string-escape"))
+                n = utils.clean_hanging_newline(n)
+                editor.walker.set_current_value(n, False)
+                editor.walker._modified()
+        elif key in ["enter"]:
+            editor.walker.start_edit()
+        else:
+            return key
+
+
+class SubgridColumn:
+    def __init__(self, heading, subeditor):
+        self.heading = heading
+        self.subeditor = subeditor
+
+    def text(self, obj):
+        p = http_cookies._format_pairs(obj, sep="\n")
+        return urwid.Text(p)
+
+    def blank(self):
+        return []
+
+    def keypress(self, key, editor):
+        if key in "rRe":
+            signals.status_message.send(
+                self,
+                message = "Press enter to edit this field.",
+                expire = 1000
+            )
+            return
+        elif key in ["enter"]:
+            editor.master.view_grideditor(
+                self.subeditor(
+                    editor.master,
+                    editor.walker.get_current_value(),
+                    editor.set_subeditor_value,
+                    editor.walker.focus,
+                    editor.walker.focus_col
+                )
+            )
+        else:
+            return key
+
+
+class SEscaped(urwid.WidgetWrap):
+    def __init__(self, txt):
         txt = txt.encode("string-escape")
         w = urwid.Text(txt, wrap="any")
-        if focused:
-            if error:
-                w = urwid.AttrWrap(w, "focusfield_error")
-            else:
-                w = urwid.AttrWrap(w, "focusfield")
-        elif error:
-            w = urwid.AttrWrap(w, "field_error")
         urwid.WidgetWrap.__init__(self, w)
 
     def get_text(self):
@@ -50,7 +118,7 @@ class SEdit(urwid.WidgetWrap):
         urwid.WidgetWrap.__init__(self, w)
 
     def get_text(self):
-        return self._w.get_text()[0]
+        return self._w.get_text()[0].strip()
 
     def selectable(self):
         return True
@@ -67,9 +135,15 @@ class GridRow(urwid.WidgetWrap):
                 self.editing = SEdit(v)
                 self.fields.append(self.editing)
             else:
-                self.fields.append(
+                w = self.editor.columns[i].text(v)
-                    SText(v, True if focused == i else False, i in errors)
+                if focused == i:
-                )
+                    if i in errors:
+                        w = urwid.AttrWrap(w, "focusfield_error")
+                    else:
+                        w = urwid.AttrWrap(w, "focusfield")
+                elif i in errors:
+                    w = urwid.AttrWrap(w, "field_error")
+                self.fields.append(w)
 
         fspecs = self.fields[:]
         if len(self.fields) > 1:
@@ -101,6 +175,7 @@ class GridWalker(urwid.ListWalker):
         and errors is a set with an entry of each offset in rows that is an
         error.
     """
+
     def __init__(self, lst, editor):
         self.lst = [(i, set([])) for i in lst]
         self.editor = editor
@@ -125,31 +200,43 @@ class GridWalker(urwid.ListWalker):
             try:
                 val = val.decode("string-escape")
             except ValueError:
-                self.editor.master.statusbar.message(
+                signals.status_message.send(
-                    "Invalid Python-style string encoding.", 1000
+                    self,
+                    message = "Invalid Python-style string encoding.",
+                    expire = 1000
                 )
                 return
         errors = self.lst[self.focus][1]
         emsg = self.editor.is_error(self.focus_col, val)
         if emsg:
-            self.editor.master.statusbar.message(emsg, 1000)
+            signals.status_message.send(message = emsg, expire = 1)
             errors.add(self.focus_col)
         else:
             errors.discard(self.focus_col)
+        self.set_value(val, self.focus, self.focus_col, errors)
 
-        row = list(self.lst[self.focus][0])
+    def set_value(self, val, focus, focus_col, errors=None):
-        row[self.focus_col] = val
+        if not errors:
-        self.lst[self.focus] = [tuple(row), errors]
+            errors = set([])
+        row = list(self.lst[focus][0])
+        row[focus_col] = val
+        self.lst[focus] = [tuple(row), errors]
+        self._modified()
 
     def delete_focus(self):
         if self.lst:
             del self.lst[self.focus]
-            self.focus = min(len(self.lst)-1, self.focus)
+            self.focus = min(len(self.lst) - 1, self.focus)
             self._modified()
 
     def _insert(self, pos):
         self.focus = pos
-        self.lst.insert(self.focus, [[""]*self.editor.columns, set([])])
+        self.lst.insert(
+            self.focus,
+            [
+                [c.blank() for c in self.editor.columns], set([])
+            ]
+        )
         self.focus_col = 0
         self.start_edit()
 
@@ -160,16 +247,17 @@ class GridWalker(urwid.ListWalker):
         return self._insert(min(self.focus + 1, len(self.lst)))
 
     def start_edit(self):
-        if self.lst:
+        col = self.editor.columns[self.focus_col]
+        if self.lst and not col.subeditor:
             self.editing = GridRow(
                 self.focus_col, True, self.editor, self.lst[self.focus]
             )
-            self.editor.master.statusbar.update(footer_editing)
+            self.editor.master.loop.widget.footer.update(FOOTER_EDITING)
             self._modified()
 
     def stop_edit(self):
         if self.editing:
-            self.editor.master.statusbar.update(footer)
+            self.editor.master.loop.widget.footer.update(FOOTER)
             self.set_current_value(self.editing.get_edit_value(), False)
             self.editing = False
             self._modified()
@@ -179,14 +267,14 @@ class GridWalker(urwid.ListWalker):
         self._modified()
 
     def right(self):
-        self.focus_col = min(self.focus_col + 1, self.editor.columns-1)
+        self.focus_col = min(self.focus_col + 1, len(self.editor.columns) - 1)
         self._modified()
 
     def tab_next(self):
         self.stop_edit()
-        if self.focus_col < self.editor.columns-1:
+        if self.focus_col < len(self.editor.columns) - 1:
             self.focus_col += 1
-        elif self.focus != len(self.lst)-1:
+        elif self.focus != len(self.lst) - 1:
             self.focus_col = 0
             self.focus += 1
         self._modified()
@@ -207,16 +295,17 @@ class GridWalker(urwid.ListWalker):
     def set_focus(self, focus):
         self.stop_edit()
         self.focus = focus
+        self._modified()
 
     def get_next(self, pos):
-        if pos+1 >= len(self.lst):
+        if pos + 1 >= len(self.lst):
             return None, None
-        return GridRow(None, False, self.editor, self.lst[pos+1]), pos+1
+        return GridRow(None, False, self.editor, self.lst[pos + 1]), pos + 1
 
     def get_prev(self, pos):
-        if pos-1 < 0:
+        if pos - 1 < 0:
             return None, None
-        return GridRow(None, False, self.editor, self.lst[pos-1]), pos-1
+        return GridRow(None, False, self.editor, self.lst[pos - 1]), pos - 1
 
 
 class GridListBox(urwid.ListBox):
@@ -231,17 +320,16 @@ FIRST_WIDTH_MIN = 20
 class GridEditor(urwid.WidgetWrap):
     title = None
     columns = None
-    headings = None
 
     def __init__(self, master, value, callback, *cb_args, **cb_kwargs):
-        value = copy.deepcopy(value)
+        value = self.data_in(copy.deepcopy(value))
         self.master, self.value, self.callback = master, value, callback
         self.cb_args, self.cb_kwargs = cb_args, cb_kwargs
 
         first_width = 20
         if value:
             for r in value:
-                assert len(r) == self.columns
+                assert len(r) == len(self.columns)
                 first_width = max(len(r), first_width)
         self.first_width = min(first_width, FIRST_WIDTH_MAX)
 
@@ -250,9 +338,9 @@ class GridEditor(urwid.WidgetWrap):
         title = urwid.AttrWrap(title, "heading")
 
         headings = []
-        for i, h in enumerate(self.headings):
+        for i, col in enumerate(self.columns):
-            c = urwid.Text(h)
+            c = urwid.Text(col.heading)
-            if i == 0 and len(self.headings) > 1:
+            if i == 0 and len(self.columns) > 1:
                 headings.append(("fixed", first_width + 2, c))
             else:
                 headings.append(c)
@@ -268,7 +356,7 @@ class GridEditor(urwid.WidgetWrap):
             self.lb,
             header = urwid.Pile([title, h])
         )
-        self.master.statusbar.update("")
+        self.master.loop.widget.footer.update("")
         self.show_empty_msg()
 
     def show_empty_msg(self):
@@ -300,9 +388,12 @@ class GridEditor(urwid.WidgetWrap):
                 d = file(p, "rb").read()
                 self.walker.set_current_value(d, unescaped)
                 self.walker._modified()
-            except IOError, v:
+            except IOError as v:
                 return str(v)
 
+    def set_subeditor_value(self, val, focus, focus_col):
+        self.walker.set_value(val, focus, focus_col)
+
     def keypress(self, size, key):
         if self.walker.editing:
             if key in ["esc"]:
@@ -317,13 +408,18 @@ class GridEditor(urwid.WidgetWrap):
             return None
 
         key = common.shortcuts(key)
+        column = self.columns[self.walker.focus_col]
         if key in ["q", "esc"]:
             res = []
             for i in self.walker.lst:
-                if not i[1] and any([x.strip() for x in i[0]]):
+                if not i[1] and any([x for x in i[0]]):
                     res.append(i[0])
-            self.callback(res, *self.cb_args, **self.cb_kwargs)
+            self.callback(self.data_out(res), *self.cb_args, **self.cb_kwargs)
-            self.master.pop_view()
+            signals.pop_view_state.send(self)
+        elif key == "G":
+            self.walker.set_focus(0)
+        elif key == "g":
+            self.walker.set_focus(len(self.walker.lst) - 1)
         elif key in ["h", "left"]:
             self.walker.left()
         elif key in ["l", "right"]:
@@ -336,26 +432,22 @@ class GridEditor(urwid.WidgetWrap):
             self.walker.insert()
         elif key == "d":
             self.walker.delete_focus()
-        elif key == "r":
+        elif column.keypress(key, self) and not self.handle_key(key):
-            if self.walker.get_current_value() is not None:
-                self.master.path_prompt("Read file: ", "", self.read_file)
-        elif key == "R":
-            if self.walker.get_current_value() is not None:
-                self.master.path_prompt(
-                    "Read unescaped file: ", "", self.read_file, True
-                )
-        elif key == "e":
-            o = self.walker.get_current_value()
-            if o is not None:
-                n = self.master.spawn_editor(o.encode("string-escape"))
-                n = utils.clean_hanging_newline(n)
-                self.walker.set_current_value(n, False)
-                self.walker._modified()
-        elif key in ["enter"]:
-            self.walker.start_edit()
-        elif not self.handle_key(key):
             return self._w.keypress(size, key)
 
+    def data_out(self, data):
+        """
+            Called on raw list data, before data is returned through the
+            callback.
+        """
+        return data
+
+    def data_in(self, data):
+        """
+            Called to prepare provided data.
+        """
+        return data
+
     def is_error(self, col, val):
         """
             Return False, or a string error message.
@@ -373,10 +465,10 @@ class GridEditor(urwid.WidgetWrap):
             ("a", "add row after cursor"),
             ("d", "delete row"),
             ("e", "spawn external editor on current field"),
-            ("q", "return to flow view"),
+            ("q", "save changes and exit editor"),
             ("r", "read value from file"),
             ("R", "read unescaped value from file"),
-            ("esc", "return to flow view/exit field edit mode"),
+            ("esc", "save changes and exit editor"),
             ("tab", "next field"),
             ("enter", "edit field"),
         ]
@@ -396,14 +488,18 @@ class GridEditor(urwid.WidgetWrap):
 
 class QueryEditor(GridEditor):
     title = "Editing query"
-    columns = 2
+    columns = [
-    headings = ("Key", "Value")
+        TextColumn("Key"),
+        TextColumn("Value")
+    ]
 
 
 class HeaderEditor(GridEditor):
     title = "Editing headers"
-    columns = 2
+    columns = [
-    headings = ("Key", "Value")
+        TextColumn("Key"),
+        TextColumn("Value")
+    ]
 
     def make_help(self):
         h = GridEditor.make_help(self)
@@ -431,24 +527,29 @@ class HeaderEditor(GridEditor):
 
     def handle_key(self, key):
         if key == "U":
-            self.master.prompt_onekey(
+            signals.status_prompt_onekey.send(
-                "Add User-Agent header:",
+                prompt = "Add User-Agent header:",
-                [(i[0], i[1]) for i in http_uastrings.UASTRINGS],
+                keys = [(i[0], i[1]) for i in http_uastrings.UASTRINGS],
-                self.set_user_agent,
+                callback = self.set_user_agent,
             )
             return True
 
 
 class URLEncodedFormEditor(GridEditor):
     title = "Editing URL-encoded form"
-    columns = 2
+    columns = [
-    headings = ("Key", "Value")
+        TextColumn("Key"),
+        TextColumn("Value")
+    ]
 
 
 class ReplaceEditor(GridEditor):
     title = "Editing replacement patterns"
-    columns = 3
+    columns = [
-    headings = ("Filter", "Regex", "Replacement")
+        TextColumn("Filter"),
+        TextColumn("Regex"),
+        TextColumn("Replacement"),
+    ]
 
     def is_error(self, col, val):
         if col == 0:
@@ -464,8 +565,11 @@ class ReplaceEditor(GridEditor):
 
 class SetHeadersEditor(GridEditor):
     title = "Editing header set patterns"
-    columns = 3
+    columns = [
-    headings = ("Filter", "Header", "Value")
+        TextColumn("Filter"),
+        TextColumn("Header"),
+        TextColumn("Value"),
+    ]
 
     def is_error(self, col, val):
         if col == 0:
@@ -500,39 +604,105 @@ class SetHeadersEditor(GridEditor):
 
     def handle_key(self, key):
         if key == "U":
-            self.master.prompt_onekey(
+            signals.status_prompt_onekey.send(
-                "Add User-Agent header:",
+                prompt = "Add User-Agent header:",
-                [(i[0], i[1]) for i in http_uastrings.UASTRINGS],
+                keys = [(i[0], i[1]) for i in http_uastrings.UASTRINGS],
-                self.set_user_agent,
+                callback = self.set_user_agent,
             )
             return True
 
 
 class PathEditor(GridEditor):
     title = "Editing URL path components"
-    columns = 1
+    columns = [
-    headings = ("Component",)
+        TextColumn("Component"),
+    ]
+
+    def data_in(self, data):
+        return [[i] for i in data]
+
+    def data_out(self, data):
+        return [i[0] for i in data]
 
 
 class ScriptEditor(GridEditor):
     title = "Editing scripts"
-    columns = 1
+    columns = [
-    headings = ("Command",)
+        TextColumn("Command"),
+    ]
 
     def is_error(self, col, val):
         try:
             script.Script.parse_command(val)
-        except script.ScriptError, v:
+        except script.ScriptError as v:
             return str(v)
 
 
 class HostPatternEditor(GridEditor):
     title = "Editing host patterns"
-    columns = 1
+    columns = [
-    headings = ("Regex (matched on hostname:port / ip:port)",)
+        TextColumn("Regex (matched on hostname:port / ip:port)")
+    ]
 
     def is_error(self, col, val):
         try:
             re.compile(val, re.IGNORECASE)
         except re.error as e:
             return "Invalid regex: %s" % str(e)
+
+    def data_in(self, data):
+        return [[i] for i in data]
+
+    def data_out(self, data):
+        return [i[0] for i in data]
+
+
+class CookieEditor(GridEditor):
+    title = "Editing request Cookie header"
+    columns = [
+        TextColumn("Name"),
+        TextColumn("Value"),
+    ]
+
+
+class CookieAttributeEditor(GridEditor):
+    title = "Editing Set-Cookie attributes"
+    columns = [
+        TextColumn("Name"),
+        TextColumn("Value"),
+    ]
+
+    def data_out(self, data):
+        ret = []
+        for i in data:
+            if not i[1]:
+                ret.append([i[0], None])
+            else:
+                ret.append(i)
+        return ret
+
+
+class SetCookieEditor(GridEditor):
+    title = "Editing response SetCookie header"
+    columns = [
+        TextColumn("Name"),
+        TextColumn("Value"),
+        SubgridColumn("Attributes", CookieAttributeEditor),
+    ]
+
+    def data_in(self, data):
+        flattened = []
+        for k, v in data.items():
+            flattened.append([k, v[0], v[1].lst])
+        return flattened
+
+    def data_out(self, data):
+        vals = []
+        for i in data:
+            vals.append(
+                [
+                    i[0],
+                    [i[1], odict.ODictCaseless(i[2])]
+                ]
+            )
+        return odict.ODict(vals)

libmproxy/console/help.py

@@ -2,18 +2,17 @@ from __future__ import absolute_import
 
 import urwid
 
-from . import common
+from . import common, signals
 from .. import filt, version
 
 footer = [
-    ("heading", 'mitmproxy v%s '%version.VERSION),
+    ("heading", 'mitmproxy v%s ' % version.VERSION),
     ('heading_key', "q"), ":back ",
 ]
 
 
 class HelpView(urwid.ListBox):
-    def __init__(self, master, help_context, state):
+    def __init__(self, help_context):
-        self.master, self.state = master, state
         self.help_context = help_context or []
         urwid.ListBox.__init__(
             self,
@@ -29,101 +28,26 @@ class HelpView(urwid.ListBox):
         keys = [
             ("j, k", "down, up"),
             ("h, l", "left, right (in some contexts)"),
+            ("g, G", "go to end, beginning"),
             ("space", "page down"),
             ("pg up/down", "page up/down"),
             ("arrows", "up, down, left, right"),
         ]
-        text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
+        text.extend(
+            common.format_keyvals(
+                keys,
+                key="key",
+                val="text",
+                indent=4))
 
         text.append(urwid.Text([("head", "\n\nGlobal keys:\n")]))
         keys = [
             ("c", "client replay"),
-            ("H", "edit global header set patterns"),
-            ("I", "set ignore pattern"),
             ("i", "set interception pattern"),
-            ("M", "change global default display mode"),
+            ("o", "options"),
-                (None,
+            ("q", "quit / return to previous page"),
-                    common.highlight_key("automatic", "a") +
-                    [("text", ": automatic detection")]
-                ),
-                (None,
-                    common.highlight_key("hex", "e") +
-                    [("text", ": Hex")]
-                ),
-                (None,
-                    common.highlight_key("html", "h") +
-                    [("text", ": HTML")]
-                ),
-                (None,
-                    common.highlight_key("image", "i") +
-                    [("text", ": Image")]
-                ),
-                (None,
-                    common.highlight_key("javascript", "j") +
-                    [("text", ": JavaScript")]
-                ),
-                (None,
-                    common.highlight_key("json", "s") +
-                    [("text", ": JSON")]
-                ),
-                (None,
-                    common.highlight_key("css", "c") +
-                    [("text", ": CSS")]
-                ),
-                (None,
-                    common.highlight_key("urlencoded", "u") +
-                    [("text", ": URL-encoded data")]
-                ),
-                (None,
-                    common.highlight_key("raw", "r") +
-                    [("text", ": raw data")]
-                ),
-                (None,
-                    common.highlight_key("xml", "x") +
-                    [("text", ": XML")]
-                ),
-                (None,
-                    common.highlight_key("wbxml", "w") +
-                    [("text", ": WBXML")]
-                ),
-                (None,
-                    common.highlight_key("amf", "f") +
-                    [("text", ": AMF (requires PyAMF)")]
-                ),
-            ("o", "toggle options:"),
-                (None,
-                    common.highlight_key("anticache", "a") +
-                    [("text", ": prevent cached responses")]
-                ),
-                (None,
-                    common.highlight_key("anticomp", "c") +
-                    [("text", ": prevent compressed responses")]
-                ),
-                (None,
-                    common.highlight_key("showhost", "h") +
-                    [("text", ": use Host header for URL display")]
-                ),
-                (None,
-                    common.highlight_key("killextra", "k") +
-                    [("text", ": kill requests not part of server replay")]
-                ),
-                (None,
-                    common.highlight_key("norefresh", "n") +
-                    [("text", ": disable server replay response refresh")]
-                ),
-                (None,
-                    common.highlight_key("upstream certs", "u") +
-                    [("text", ": sniff cert info from upstream server")]
-                ),
-
-            ("q", "quit / return to flow list"),
             ("Q", "quit without confirm prompt"),
-            ("R", "edit replacement patterns"),
-            ("s", "add/remove scripts"),
             ("S", "server replay"),
-            ("t", "set sticky cookie expression"),
-            ("T", "set tcp proxying pattern"),
-            ("u", "set sticky auth expression"),
         ]
         text.extend(
             common.format_keyvals(keys, key="key", val="text", indent=4)
@@ -133,15 +57,15 @@ class HelpView(urwid.ListBox):
         f = []
         for i in filt.filt_unary:
             f.append(
-                ("~%s"%i.code, i.help)
+                ("~%s" % i.code, i.help)
             )
         for i in filt.filt_rex:
             f.append(
-                ("~%s regex"%i.code, i.help)
+                ("~%s regex" % i.code, i.help)
             )
         for i in filt.filt_int:
             f.append(
-                ("~%s int"%i.code, i.help)
+                ("~%s int" % i.code, i.help)
             )
         f.sort()
         f.extend(
@@ -156,7 +80,7 @@ class HelpView(urwid.ListBox):
 
         text.append(
             urwid.Text(
-               [
+                [
                     "\n",
                     ("text", "    Regexes are Python-style.\n"),
                     ("text", "    Regexes can be specified as quoted strings.\n"),
@@ -164,13 +88,13 @@ class HelpView(urwid.ListBox):
                     ("text", "    Expressions with no operators are regex matches against URL.\n"),
                     ("text", "    Default binary operator is &.\n"),
                     ("head", "\n    Examples:\n"),
-               ]
+                ]
             )
         )
         examples = [
-                ("google\.com", "Url containing \"google.com"),
+            ("google\.com", "Url containing \"google.com"),
-                ("~q ~b test", "Requests where body contains \"test\""),
+            ("~q ~b test", "Requests where body contains \"test\""),
-                ("!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."),
+            ("!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."),
         ]
         text.extend(
             common.format_keyvals(examples, key="key", val="text", indent=4)
@@ -180,11 +104,12 @@ class HelpView(urwid.ListBox):
     def keypress(self, size, key):
         key = common.shortcuts(key)
         if key == "q":
-            self.master.statusbar = self.state[0]
+            signals.pop_view_state.send(self)
-            self.master.body = self.state[1]
-            self.master.header = self.state[2]
-            self.master.loop.widget = self.master.make_view()
             return None
         elif key == "?":
             key = None
+        elif key == "G":
+            self.set_focus(0)
+        elif key == "g":
+            self.set_focus(len(self.body.contents))
         return urwid.ListBox.keypress(self, size, key)

libmproxy/console/options.py

@@ -0,0 +1,269 @@
+import urwid
+
+from . import common, signals, grideditor, contentview
+from . import select, palettes
+
+footer = [
+    ('heading_key', "enter/space"), ":toggle ",
+    ('heading_key', "C"), ":clear all ",
+]
+
+
+def _mkhelp():
+    text = []
+    keys = [
+        ("enter/space", "activate option"),
+        ("C", "clear all options"),
+    ]
+    text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
+    return text
+help_context = _mkhelp()
+
+
+class Options(urwid.WidgetWrap):
+    def __init__(self, master):
+        self.master = master
+        self.lb = select.Select(
+            [
+                select.Heading("Traffic Manipulation"),
+                select.Option(
+                    "Header Set Patterns",
+                    "H",
+                    lambda: master.setheaders.count(),
+                    self.setheaders
+                ),
+                select.Option(
+                    "Ignore Patterns",
+                    "I",
+                    lambda: master.server.config.check_ignore,
+                    self.ignorepatterns
+                ),
+                select.Option(
+                    "Replacement Patterns",
+                    "R",
+                    lambda: master.replacehooks.count(),
+                    self.replacepatterns
+                ),
+                select.Option(
+                    "Scripts",
+                    "S",
+                    lambda: master.scripts,
+                    self.scripts
+                ),
+
+                select.Heading("Interface"),
+                select.Option(
+                    "Default Display Mode",
+                    "M",
+                    self.has_default_displaymode,
+                    self.default_displaymode
+                ),
+                select.Option(
+                    "Palette",
+                    "P",
+                    lambda: self.master.palette != palettes.DEFAULT,
+                    self.palette
+                ),
+                select.Option(
+                    "Show Host",
+                    "w",
+                    lambda: master.showhost,
+                    self.toggle_showhost
+                ),
+
+                select.Heading("Network"),
+                select.Option(
+                    "No Upstream Certs",
+                    "U",
+                    lambda: master.server.config.no_upstream_cert,
+                    self.toggle_upstream_cert
+                ),
+                select.Option(
+                    "TCP Proxying",
+                    "T",
+                    lambda: master.server.config.check_tcp,
+                    self.tcp_proxy
+                ),
+
+                select.Heading("Utility"),
+                select.Option(
+                    "Anti-Cache",
+                    "a",
+                    lambda: master.anticache,
+                    self.toggle_anticache
+                ),
+                select.Option(
+                    "Anti-Compression",
+                    "o",
+                    lambda: master.anticomp,
+                    self.toggle_anticomp
+                ),
+                select.Option(
+                    "Kill Extra",
+                    "x",
+                    lambda: master.killextra,
+                    self.toggle_killextra
+                ),
+                select.Option(
+                    "No Refresh",
+                    "f",
+                    lambda: not master.refresh_server_playback,
+                    self.toggle_refresh_server_playback
+                ),
+                select.Option(
+                    "Sticky Auth",
+                    "A",
+                    lambda: master.stickyauth_txt,
+                    self.sticky_auth
+                ),
+                select.Option(
+                    "Sticky Cookies",
+                    "t",
+                    lambda: master.stickycookie_txt,
+                    self.sticky_cookie
+                ),
+            ]
+        )
+        title = urwid.Text("Options")
+        title = urwid.Padding(title, align="left", width=("relative", 100))
+        title = urwid.AttrWrap(title, "heading")
+        self._w = urwid.Frame(
+            self.lb,
+            header = title
+        )
+        self.master.loop.widget.footer.update("")
+        signals.update_settings.connect(self.sig_update_settings)
+
+    def sig_update_settings(self, sender):
+        self.lb.walker._modified()
+
+    def keypress(self, size, key):
+        if key == "C":
+            self.clearall()
+            return None
+        return super(self.__class__, self).keypress(size, key)
+
+    def clearall(self):
+        self.master.anticache = False
+        self.master.anticomp = False
+        self.master.killextra = False
+        self.master.showhost = False
+        self.master.refresh_server_playback = True
+        self.master.server.config.no_upstream_cert = False
+        self.master.setheaders.clear()
+        self.master.replacehooks.clear()
+        self.master.set_ignore_filter([])
+        self.master.set_tcp_filter([])
+        self.master.scripts = []
+        self.master.set_stickyauth(None)
+        self.master.set_stickycookie(None)
+        self.master.state.default_body_view = contentview.get("Auto")
+
+        signals.update_settings.send(self)
+        signals.status_message.send(
+            message = "All select.Options cleared",
+            expire = 1
+        )
+
+    def toggle_anticache(self):
+        self.master.anticache = not self.master.anticache
+
+    def toggle_anticomp(self):
+        self.master.anticomp = not self.master.anticomp
+
+    def toggle_killextra(self):
+        self.master.killextra = not self.master.killextra
+
+    def toggle_showhost(self):
+        self.master.showhost = not self.master.showhost
+
+    def toggle_refresh_server_playback(self):
+        self.master.refresh_server_playback = not self.master.refresh_server_playback
+
+    def toggle_upstream_cert(self):
+        self.master.server.config.no_upstream_cert = not self.master.server.config.no_upstream_cert
+        signals.update_settings.send(self)
+
+    def setheaders(self):
+        def _set(*args, **kwargs):
+            self.master.setheaders.set(*args, **kwargs)
+            signals.update_settings.send(self)
+        self.master.view_grideditor(
+            grideditor.SetHeadersEditor(
+                self.master,
+                self.master.setheaders.get_specs(),
+                _set
+            )
+        )
+
+    def ignorepatterns(self):
+        def _set(ignore):
+            self.master.set_ignore_filter(ignore)
+            signals.update_settings.send(self)
+        self.master.view_grideditor(
+            grideditor.HostPatternEditor(
+                self.master,
+                self.master.get_ignore_filter(),
+                _set
+            )
+        )
+
+    def replacepatterns(self):
+        def _set(*args, **kwargs):
+            self.master.replacehooks.set(*args, **kwargs)
+            signals.update_settings.send(self)
+        self.master.view_grideditor(
+            grideditor.ReplaceEditor(
+                self.master,
+                self.master.replacehooks.get_specs(),
+                _set
+            )
+        )
+
+    def scripts(self):
+        self.master.view_grideditor(
+            grideditor.ScriptEditor(
+                self.master,
+                [[i.command] for i in self.master.scripts],
+                self.master.edit_scripts
+            )
+        )
+
+    def default_displaymode(self):
+        signals.status_prompt_onekey.send(
+            prompt = "Global default display mode",
+            keys = contentview.view_prompts,
+            callback = self.master.change_default_display_mode
+        )
+
+    def has_default_displaymode(self):
+        return self.master.state.default_body_view.name != "Auto"
+
+    def tcp_proxy(self):
+        def _set(tcp):
+            self.master.set_tcp_filter(tcp)
+            signals.update_settings.send(self)
+        self.master.view_grideditor(
+            grideditor.HostPatternEditor(
+                self.master,
+                self.master.get_tcp_filter(),
+                _set
+            )
+        )
+
+    def sticky_auth(self):
+        signals.status_prompt.send(
+            prompt = "Sticky auth filter",
+            text = self.master.stickyauth_txt,
+            callback = self.master.set_stickyauth
+        )
+
+    def sticky_cookie(self):
+        signals.status_prompt.send(
+            prompt = "Sticky cookie filter",
+            text = self.master.stickycookie_txt,
+            callback = self.master.set_stickycookie
+        )
+
+    def palette(self):
+        self.master.view_palette_picker()

libmproxy/console/palettepicker.py

@@ -0,0 +1,81 @@
+import urwid
+
+from . import select, common, palettes, signals
+
+footer = [
+    ('heading_key', "enter/space"), ":select",
+]
+
+
+def _mkhelp():
+    text = []
+    keys = [
+        ("enter/space", "select"),
+    ]
+    text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
+    return text
+help_context = _mkhelp()
+
+
+class PalettePicker(urwid.WidgetWrap):
+    def __init__(self, master):
+        self.master = master
+        low, high = [], []
+        for k, v in palettes.palettes.items():
+            if v.high:
+                high.append(k)
+            else:
+                low.append(k)
+        high.sort()
+        low.sort()
+
+        options = [
+            select.Heading("High Colour")
+        ]
+
+        def mkopt(name):
+            return select.Option(
+                i,
+                None,
+                lambda: self.master.palette == name,
+                lambda: self.select(name)
+            )
+
+        for i in high:
+            options.append(mkopt(i))
+        options.append(select.Heading("Low Colour"))
+        for i in low:
+            options.append(mkopt(i))
+
+        options.extend(
+            [
+                select.Heading("Options"),
+                select.Option(
+                    "Transparent",
+                    "T",
+                    lambda: master.palette_transparent,
+                    self.toggle_palette_transparent
+                )
+            ]
+        )
+
+        self.lb = select.Select(options)
+        title = urwid.Text("Palettes")
+        title = urwid.Padding(title, align="left", width=("relative", 100))
+        title = urwid.AttrWrap(title, "heading")
+        self._w = urwid.Frame(
+            self.lb,
+            header = title
+        )
+        signals.update_settings.connect(self.sig_update_settings)
+
+    def sig_update_settings(self, sender):
+        self.lb.walker._modified()
+
+    def select(self, name):
+        self.master.set_palette(name)
+
+    def toggle_palette_transparent(self):
+        self.master.palette_transparent = not self.master.palette_transparent
+        self.master.set_palette(self.master.palette)
+        signals.update_settings.send(self)

libmproxy/console/palettes.py

@@ -1,4 +1,3 @@
-
 # Low-color themes should ONLY use the standard foreground and background
 # colours listed here:
 #
@@ -6,9 +5,9 @@
 #
 
 
-
 class Palette:
     _fields = [
+        'background',
         'title',
 
         # Status bar & heading
@@ -17,6 +16,10 @@ class Palette:
         # Help
         'key', 'head', 'text',
 
+        # Options
+        'option_selected', 'option_active', 'option_active_selected',
+        'option_selected_key',
+
         # List and Connections
         'method', 'focus',
         'code_200', 'code_300', 'code_400', 'code_500', 'code_other',
@@ -31,15 +34,33 @@ class Palette:
     ]
     high = None
 
-    def palette(self):
+    def palette(self, transparent):
         l = []
+        highback, lowback = None, None
+        if not transparent:
+            if self.high and self.high.get("background"):
+                highback = self.high["background"][1]
+            lowback = self.low["background"][1]
+
         for i in self._fields:
-            v = [i]
+            if transparent and i == "background":
-            v.extend(self.low[i])
+                l.append(["background", "default", "default"])
-            if self.high and i in self.high:
+            else:
-                v.append(None)
+                v = [i]
-                v.extend(self.high[i])
+                low = list(self.low[i])
-            l.append(tuple(v))
+                if lowback and low[1] == "default":
+                    low[1] = lowback
+                v.extend(low)
+                if self.high and i in self.high:
+                    v.append(None)
+                    high = list(self.high[i])
+                    if highback and high[1] == "default":
+                        high[1] = highback
+                    v.extend(high)
+                elif highback and self.low[i][1] == "default":
+                    high = [None, low[0], highback]
+                    v.extend(high)
+                l.append(tuple(v))
         return l
 
 
@@ -48,18 +69,25 @@ class LowDark(Palette):
         Low-color dark background
     """
     low = dict(
+        background = ('white', 'black'),
         title = ('white,bold', 'default'),
 
         # Status bar & heading
-        heading = ('light gray', 'dark blue'),
+        heading = ('white', 'dark blue'),
         heading_key = ('light cyan', 'dark blue'),
-        heading_inactive = ('white', 'dark gray'),
+        heading_inactive = ('dark gray', 'light gray'),
 
         # Help
         key = ('light cyan', 'default'),
         head = ('white,bold', 'default'),
         text = ('light gray', 'default'),
 
+        # Options
+        option_selected = ('black', 'light gray'),
+        option_selected_key = ('light cyan', 'light gray'),
+        option_active = ('light red', 'default'),
+        option_active_selected = ('light red', 'light gray'),
+
         # List and Connections
         method = ('dark cyan', 'default'),
         focus = ('yellow', 'default'),
@@ -92,6 +120,10 @@ class Dark(LowDark):
     high = dict(
         heading_inactive = ('g58', 'g11'),
         intercept = ('#f60', 'default'),
+
+        option_selected = ('g85', 'g45'),
+        option_selected_key = ('light cyan', 'g50'),
+        option_active_selected = ('light red', 'g50'),
     )
 
 
@@ -100,18 +132,25 @@ class LowLight(Palette):
         Low-color light background
     """
     low = dict(
-        title = ('dark magenta,bold', 'light blue'),
+        background = ('black', 'white'),
+        title = ('dark magenta', 'default'),
 
         # Status bar & heading
-        heading = ('light gray', 'dark blue'),
+        heading = ('white', 'black'),
-        heading_key = ('light cyan', 'dark blue'),
+        heading_key = ('dark blue', 'black'),
         heading_inactive = ('black', 'light gray'),
 
         # Help
-        key = ('dark blue,bold', 'default'),
+        key = ('dark blue', 'default'),
-        head = ('black,bold', 'default'),
+        head = ('black', 'default'),
         text = ('dark gray', 'default'),
 
+        # Options
+        option_selected = ('black', 'light gray'),
+        option_selected_key = ('dark blue', 'light gray'),
+        option_active = ('light red', 'default'),
+        option_active_selected = ('light red', 'light gray'),
+
         # List and Connections
         method = ('dark cyan', 'default'),
         focus = ('black', 'default'),
@@ -142,10 +181,15 @@ class LowLight(Palette):
 
 class Light(LowLight):
     high = dict(
+        background = ('black', 'g100'),
         heading = ('g99', '#08f'),
         heading_key = ('#0ff,bold', '#08f'),
         heading_inactive = ('g35', 'g85'),
         replay = ('#0a0,bold', 'default'),
+
+        option_selected = ('black', 'g85'),
+        option_selected_key = ('dark blue', 'g85'),
+        option_active_selected = ('light red', 'g85'),
     )
 
 
@@ -155,10 +199,10 @@ sol_base03 = "h234"
 sol_base02 = "h235"
 sol_base01 = "h240"
 sol_base00 = "h241"
-sol_base0  = "h244"
+sol_base0 = "h244"
-sol_base1  = "h245"
+sol_base1 = "h245"
-sol_base2  = "h254"
+sol_base2 = "h254"
-sol_base3  = "h230"
+sol_base3 = "h230"
 sol_yellow = "h136"
 sol_orange = "h166"
 sol_red = "h160"
@@ -167,9 +211,12 @@ sol_violet = "h61"
 sol_blue = "h33"
 sol_cyan = "h37"
 sol_green = "h64"
+
+
 class SolarizedLight(LowLight):
     high = dict(
-        title = (sol_blue, 'default'),
+        background = (sol_base00, sol_base3),
+        title = (sol_cyan, 'default'),
         text = (sol_base00, 'default'),
 
         # Status bar & heading
@@ -181,6 +228,12 @@ class SolarizedLight(LowLight):
         key = (sol_blue, 'default',),
         head = (sol_base00, 'default'),
 
+        # Options
+        option_selected = (sol_base03, sol_base2),
+        option_selected_key = (sol_blue, sol_base2),
+        option_active = (sol_orange, 'default'),
+        option_active_selected = (sol_orange, sol_base2),
+
         # List and Connections
         method = (sol_cyan, 'default'),
         focus = (sol_base01, 'default'),
@@ -193,7 +246,7 @@ class SolarizedLight(LowLight):
 
         error = (sol_red, 'default'),
 
-        header = (sol_base01, 'default'),
+        header = (sol_blue, 'default'),
         highlight = (sol_base01, 'default'),
         intercept = (sol_red, 'default',),
         replay = (sol_green, 'default',),
@@ -211,17 +264,24 @@ class SolarizedLight(LowLight):
 
 class SolarizedDark(LowDark):
     high = dict(
+        background = (sol_base2, sol_base03),
         title = (sol_blue, 'default'),
-        text = (sol_base0, 'default'),
+        text = (sol_base1, 'default'),
 
         # Status bar & heading
-        heading = (sol_base03, sol_base1),
+        heading = (sol_base2, sol_base01),
-        heading_key = (sol_blue+",bold", sol_base1),
+        heading_key = (sol_blue + ",bold", sol_base01),
         heading_inactive = (sol_base1, sol_base02),
 
         # Help
         key = (sol_blue, 'default',),
-        head = (sol_base00, 'default'),
+        head = (sol_base2, 'default'),
+
+        # Options
+        option_selected = (sol_base03, sol_base00),
+        option_selected_key = (sol_blue, sol_base00),
+        option_active = (sol_orange, 'default'),
+        option_active_selected = (sol_orange, sol_base00),
 
         # List and Connections
         method = (sol_cyan, 'default'),
@@ -235,7 +295,7 @@ class SolarizedDark(LowDark):
 
         error = (sol_red, 'default'),
 
-        header = (sol_base01, 'default'),
+        header = (sol_blue, 'default'),
         highlight = (sol_base01, 'default'),
         intercept = (sol_red, 'default',),
         replay = (sol_green, 'default',),
@@ -251,6 +311,7 @@ class SolarizedDark(LowDark):
     )
 
 
+DEFAULT = "dark"
 palettes = {
     "lowlight": LowLight(),
     "lowdark": LowDark(),

libmproxy/console/pathedit.py

@@ -0,0 +1,69 @@
+import glob
+import os.path
+
+import urwid
+
+
+class _PathCompleter:
+    def __init__(self, _testing=False):
+        """
+            _testing: disables reloading of the lookup table to make testing
+            possible.
+        """
+        self.lookup, self.offset = None, None
+        self.final = None
+        self._testing = _testing
+
+    def reset(self):
+        self.lookup = None
+        self.offset = -1
+
+    def complete(self, txt):
+        """
+            Returns the next completion for txt, or None if there is no
+            completion.
+        """
+        path = os.path.expanduser(txt)
+        if not self.lookup:
+            if not self._testing:
+                # Lookup is a set of (display value, actual value) tuples.
+                self.lookup = []
+                if os.path.isdir(path):
+                    files = glob.glob(os.path.join(path, "*"))
+                    prefix = txt
+                else:
+                    files = glob.glob(path + "*")
+                    prefix = os.path.dirname(txt)
+                prefix = prefix or "./"
+                for f in files:
+                    display = os.path.join(prefix, os.path.basename(f))
+                    if os.path.isdir(f):
+                        display += "/"
+                    self.lookup.append((display, f))
+            if not self.lookup:
+                self.final = path
+                return path
+            self.lookup.sort()
+            self.offset = -1
+            self.lookup.append((txt, txt))
+        self.offset += 1
+        if self.offset >= len(self.lookup):
+            self.offset = 0
+        ret = self.lookup[self.offset]
+        self.final = ret[1]
+        return ret[0]
+
+
+class PathEdit(urwid.Edit, _PathCompleter):
+    def __init__(self, *args, **kwargs):
+        urwid.Edit.__init__(self, *args, **kwargs)
+        _PathCompleter.__init__(self)
+
+    def keypress(self, size, key):
+        if key == "tab":
+            comp = self.complete(self.get_edit_text())
+            self.set_edit_text(comp)
+            self.set_edit_pos(len(comp))
+        else:
+            self.reset()
+        return urwid.Edit.keypress(self, size, key)

libmproxy/console/searchable.py

@@ -0,0 +1,91 @@
+import urwid
+
+from . import signals
+
+
+class Highlight(urwid.AttrMap):
+    def __init__(self, t):
+        urwid.AttrMap.__init__(
+            self,
+            urwid.Text(t.text),
+            "focusfield",
+        )
+        self.backup = t
+
+
+class Searchable(urwid.ListBox):
+    def __init__(self, state, contents):
+        self.walker = urwid.SimpleFocusListWalker(contents)
+        urwid.ListBox.__init__(self, self.walker)
+        self.state = state
+        self.search_offset = 0
+        self.current_highlight = None
+        self.search_term = None
+
+    def keypress(self, size, key):
+        if key == "/":
+            signals.status_prompt.send(
+                prompt = "Search for",
+                text = "",
+                callback = self.set_search
+            )
+        elif key == "n":
+            self.find_next(False)
+        elif key == "N":
+            self.find_next(True)
+        elif key == "G":
+            self.set_focus(0)
+            self.walker._modified()
+        elif key == "g":
+            self.set_focus(len(self.walker) - 1)
+            self.walker._modified()
+        else:
+            return super(self.__class__, self).keypress(size, key)
+
+    def set_search(self, text):
+        self.state.last_search = text
+        self.search_term = text or None
+        self.find_next(False)
+
+    def set_highlight(self, offset):
+        if self.current_highlight is not None:
+            old = self.body[self.current_highlight]
+            self.body[self.current_highlight] = old.backup
+        if offset is None:
+            self.current_highlight = None
+        else:
+            self.body[offset] = Highlight(self.body[offset])
+            self.current_highlight = offset
+
+    def get_text(self, w):
+        if isinstance(w, urwid.Text):
+            return w.text
+        elif isinstance(w, Highlight):
+            return w.backup.text
+        else:
+            return None
+
+    def find_next(self, backwards):
+        if not self.search_term:
+            if self.state.last_search:
+                self.search_term = self.state.last_search
+            else:
+                self.set_highlight(None)
+                return
+        # Start search at focus + 1
+        if backwards:
+            rng = xrange(len(self.body) - 1, -1, -1)
+        else:
+            rng = xrange(1, len(self.body) + 1)
+        for i in rng:
+            off = (self.focus_position + i) % len(self.body)
+            w = self.body[off]
+            txt = self.get_text(w)
+            if txt and self.search_term in txt:
+                self.set_highlight(off)
+                self.set_focus(off, coming_from="above")
+                self.body._modified()
+                return
+        else:
+            self.set_highlight(None)
+            signals.status_message.send(message="Search not found.", expire=1)

libmproxy/console/select.py

@@ -0,0 +1,115 @@
+import urwid
+
+from . import common
+
+
+class _OptionWidget(urwid.WidgetWrap):
+    def __init__(self, option, text, shortcut, active, focus):
+        self.option = option
+        textattr = "text"
+        keyattr = "key"
+        if focus and active:
+            textattr = "option_active_selected"
+            keyattr = "option_selected_key"
+        elif focus:
+            textattr = "option_selected"
+            keyattr = "option_selected_key"
+        elif active:
+            textattr = "option_active"
+        if shortcut:
+            text = common.highlight_key(
+                text,
+                shortcut,
+                textattr = textattr,
+                keyattr = keyattr
+            )
+        opt = urwid.Text(text, align="left")
+        opt = urwid.AttrWrap(opt, textattr)
+        opt = urwid.Padding(opt, align = "center", width = 40)
+        urwid.WidgetWrap.__init__(self, opt)
+
+    def keypress(self, size, key):
+        return key
+
+    def selectable(self):
+        return True
+
+
+class OptionWalker(urwid.ListWalker):
+    def __init__(self, options):
+        urwid.ListWalker.__init__(self)
+        self.options = options
+        self.focus = 0
+
+    def set_focus(self, pos):
+        self.focus = pos
+
+    def get_focus(self):
+        return self.options[self.focus].render(True), self.focus
+
+    def get_next(self, pos):
+        if pos >= len(self.options) - 1:
+            return None, None
+        return self.options[pos + 1].render(False), pos + 1
+
+    def get_prev(self, pos):
+        if pos <= 0:
+            return None, None
+        return self.options[pos - 1].render(False), pos - 1
+
+
+class Heading:
+    def __init__(self, text):
+        self.text = text
+
+    def render(self, focus):
+        opt = urwid.Text("\n" + self.text, align="left")
+        opt = urwid.AttrWrap(opt, "title")
+        opt = urwid.Padding(opt, align = "center", width = 40)
+        return opt
+
+
+_neg = lambda: False
+
+
+class Option:
+    def __init__(self, text, shortcut, getstate=None, activate=None):
+        self.text = text
+        self.shortcut = shortcut
+        self.getstate = getstate or _neg
+        self.activate = activate or _neg
+
+    def render(self, focus):
+        return _OptionWidget(
+            self,
+            self.text,
+            self.shortcut,
+            self.getstate(),
+            focus)
+
+
+class Select(urwid.ListBox):
+    def __init__(self, options):
+        self.walker = OptionWalker(options)
+        urwid.ListBox.__init__(
+            self,
+            self.walker
+        )
+        self.options = options
+        self.keymap = {}
+        for i in options:
+            if hasattr(i, "shortcut") and i.shortcut:
+                if i.shortcut in self.keymap:
+                    raise ValueError("Duplicate shortcut key: %s" % i.shortcut)
+                self.keymap[i.shortcut] = i
+
+    def keypress(self, size, key):
+        if key == "enter" or key == " ":
+            self.get_focus()[0].option.activate()
+            return None
+        key = common.shortcuts(key)
+        if key in self.keymap:
+            self.keymap[key].activate()
+            self.set_focus(self.options.index(self.keymap[key]))
+            return None
+        return super(self.__class__, self).keypress(size, key)

libmproxy/console/signals.py

@@ -0,0 +1,32 @@
+import blinker
+
+# Show a status message in the action bar
+status_message = blinker.Signal()
+
+# Prompt for input
+status_prompt = blinker.Signal()
+
+# Prompt for a path
+status_prompt_path = blinker.Signal()
+
+# Prompt for a single keystroke
+status_prompt_onekey = blinker.Signal()
+
+# Call a callback in N seconds
+call_in = blinker.Signal()
+
+# Focus the body, footer or header of the main window
+focus = blinker.Signal()
+
+# Fired when settings change
+update_settings = blinker.Signal()
+
+# Fired when a flow changes
+flow_change = blinker.Signal()
+
+# Fired when the flow list or focus changes
+flowlist_change = blinker.Signal()
+
+# Pop and push view state onto a stack
+pop_view_state = blinker.Signal()
+push_view_state = blinker.Signal()

libmproxy/console/statusbar.py

@@ -0,0 +1,254 @@
+import os.path
+
+import urwid
+
+import netlib.utils
+from . import pathedit, signals, common
+from .. import utils
+
+
+class ActionBar(urwid.WidgetWrap):
+    def __init__(self):
+        urwid.WidgetWrap.__init__(self, None)
+        self.clear()
+        signals.status_message.connect(self.sig_message)
+        signals.status_prompt.connect(self.sig_prompt)
+        signals.status_prompt_path.connect(self.sig_path_prompt)
+        signals.status_prompt_onekey.connect(self.sig_prompt_onekey)
+
+        self.last_path = ""
+
+        self.prompting = False
+        self.onekey = False
+        self.pathprompt = False
+
+    def sig_message(self, sender, message, expire=None):
+        w = urwid.Text(message)
+        self._w = w
+        if expire:
+            def cb(*args):
+                if w == self._w:
+                    self.clear()
+            signals.call_in.send(seconds=expire, callback=cb)
+
+    def prep_prompt(self, p):
+        return p.strip() + ": "
+
+    def sig_prompt(self, sender, prompt, text, callback, args=()):
+        signals.focus.send(self, section="footer")
+        self._w = urwid.Edit(self.prep_prompt(prompt), text or "")
+        self.prompting = (callback, args)
+
+    def sig_path_prompt(self, sender, prompt, callback, args=()):
+        signals.focus.send(self, section="footer")
+        self._w = pathedit.PathEdit(
+            self.prep_prompt(prompt),
+            os.path.dirname(self.last_path)
+        )
+        self.pathprompt = True
+        self.prompting = (callback, args)
+
+    def sig_prompt_onekey(self, sender, prompt, keys, callback, args=()):
+        """
+            Keys are a set of (word, key) tuples. The appropriate key in the
+            word is highlighted.
+        """
+        signals.focus.send(self, section="footer")
+        prompt = [prompt, " ("]
+        mkup = []
+        for i, e in enumerate(keys):
+            mkup.extend(common.highlight_key(e[0], e[1]))
+            if i < len(keys) - 1:
+                mkup.append(",")
+        prompt.extend(mkup)
+        prompt.append(")? ")
+        self.onekey = set(i[1] for i in keys)
+        self._w = urwid.Edit(prompt, "")
+        self.prompting = (callback, args)
+
+    def selectable(self):
+        return True
+
+    def keypress(self, size, k):
+        if self.prompting:
+            if k == "esc":
+                self.prompt_done()
+            elif self.onekey:
+                if k == "enter":
+                    self.prompt_done()
+                elif k in self.onekey:
+                    self.prompt_execute(k)
+            elif k == "enter":
+                self.prompt_execute(self._w.get_edit_text())
+            else:
+                if common.is_keypress(k):
+                    self._w.keypress(size, k)
+                else:
+                    return k
+
+    def clear(self):
+        self._w = urwid.Text("")
+
+    def prompt_done(self):
+        self.prompting = False
+        self.onekey = False
+        self.pathprompt = False
+        signals.status_message.send(message="")
+        signals.focus.send(self, section="body")
+
+    def prompt_execute(self, txt):
+        if self.pathprompt:
+            self.last_path = txt
+        p, args = self.prompting
+        self.prompt_done()
+        msg = p(txt, *args)
+        if msg:
+            signals.status_message.send(message=msg, expire=1)
+
+
+class StatusBar(urwid.WidgetWrap):
+    def __init__(self, master, helptext):
+        self.master, self.helptext = master, helptext
+        self.ab = ActionBar()
+        self.ib = urwid.WidgetWrap(urwid.Text(""))
+        self._w = urwid.Pile([self.ib, self.ab])
+        signals.update_settings.connect(self.sig_update_settings)
+        signals.flowlist_change.connect(self.sig_update_settings)
+        self.redraw()
+
+    def sig_update_settings(self, sender):
+        self.redraw()
+
+    def keypress(self, *args, **kwargs):
+        return self.ab.keypress(*args, **kwargs)
+
+    def get_status(self):
+        r = []
+
+        if self.master.setheaders.count():
+            r.append("[")
+            r.append(("heading_key", "H"))
+            r.append("eaders]")
+        if self.master.replacehooks.count():
+            r.append("[")
+            r.append(("heading_key", "R"))
+            r.append("eplacing]")
+        if self.master.client_playback:
+            r.append("[")
+            r.append(("heading_key", "cplayback"))
+            r.append(":%s to go]" % self.master.client_playback.count())
+        if self.master.server_playback:
+            r.append("[")
+            r.append(("heading_key", "splayback"))
+            if self.master.nopop:
+                r.append(":%s in file]" % self.master.server_playback.count())
+            else:
+                r.append(":%s to go]" % self.master.server_playback.count())
+        if self.master.get_ignore_filter():
+            r.append("[")
+            r.append(("heading_key", "I"))
+            r.append("gnore:%d]" % len(self.master.get_ignore_filter()))
+        if self.master.get_tcp_filter():
+            r.append("[")
+            r.append(("heading_key", "T"))
+            r.append("CP:%d]" % len(self.master.get_tcp_filter()))
+        if self.master.state.intercept_txt:
+            r.append("[")
+            r.append(("heading_key", "i"))
+            r.append(":%s]" % self.master.state.intercept_txt)
+        if self.master.state.limit_txt:
+            r.append("[")
+            r.append(("heading_key", "l"))
+            r.append(":%s]" % self.master.state.limit_txt)
+        if self.master.stickycookie_txt:
+            r.append("[")
+            r.append(("heading_key", "t"))
+            r.append(":%s]" % self.master.stickycookie_txt)
+        if self.master.stickyauth_txt:
+            r.append("[")
+            r.append(("heading_key", "u"))
+            r.append(":%s]" % self.master.stickyauth_txt)
+        if self.master.state.default_body_view.name != "Auto":
+            r.append("[")
+            r.append(("heading_key", "M"))
+            r.append(":%s]" % self.master.state.default_body_view.name)
+
+        opts = []
+        if self.master.anticache:
+            opts.append("anticache")
+        if self.master.anticomp:
+            opts.append("anticomp")
+        if self.master.showhost:
+            opts.append("showhost")
+        if not self.master.refresh_server_playback:
+            opts.append("norefresh")
+        if self.master.killextra:
+            opts.append("killextra")
+        if self.master.server.config.no_upstream_cert:
+            opts.append("no-upstream-cert")
+        if self.master.state.follow_focus:
+            opts.append("following")
+        if self.master.stream_large_bodies:
+            opts.append(
+                "stream:%s" % netlib.utils.pretty_size(
+                    self.master.stream_large_bodies.max_size
+                )
+            )
+
+        if opts:
+            r.append("[%s]" % (":".join(opts)))
+
+        if self.master.server.config.mode in ["reverse", "upstream"]:
+            dst = self.master.server.config.mode.dst
+            scheme = "https" if dst[0] else "http"
+            if dst[1] != dst[0]:
+                scheme += "2https" if dst[1] else "http"
+            r.append("[dest:%s]" % utils.unparse_url(scheme, *dst[2:]))
+        if self.master.scripts:
+            r.append("[")
+            r.append(("heading_key", "s"))
+            r.append("cripts:%s]" % len(self.master.scripts))
+        # r.append("[lt:%0.3f]"%self.master.looptime)
+
+        if self.master.stream:
+            r.append("[W:%s]" % self.master.stream_path)
+
+        return r
+
+    def redraw(self):
+        fc = self.master.state.flow_count()
+        if self.master.state.focus is None:
+            offset = 0
+        else:
+            offset = min(self.master.state.focus + 1, fc)
+        t = [
+            ('heading', ("[%s/%s]" % (offset, fc)).ljust(9))
+        ]
+
+        if self.master.server.bound:
+            host = self.master.server.address.host
+            if host == "0.0.0.0":
+                host = "*"
+            boundaddr = "[%s:%s]" % (host, self.master.server.address.port)
+        else:
+            boundaddr = ""
+        t.extend(self.get_status())
+        status = urwid.AttrWrap(urwid.Columns([
+            urwid.Text(t),
+            urwid.Text(
+                [
+                    self.helptext,
+                    boundaddr
+                ],
+                align="right"
+            ),
+        ]), "heading")
+        self.ib._w = status
+
+    def update(self, text):
+        self.helptext = text
+        self.redraw()
+        self.master.loop.draw_screen()
+
+    def selectable(self):
+        return True

libmproxy/console/tabs.py

@@ -0,0 +1,39 @@
+import urwid
+
+
+class Tabs(urwid.WidgetWrap):
+    def __init__(self, tabs, tab_offset=0):
+        urwid.WidgetWrap.__init__(self, "")
+        self.tab_offset = tab_offset
+        self.tabs = tabs
+        self.show()
+
+    def _tab(self, content, attr):
+        p = urwid.Text(content, align="center")
+        p = urwid.Padding(p, align="center", width=("relative", 100))
+        p = urwid.AttrWrap(p, attr)
+        return p
+
+    def keypress(self, size, key):
+        if key in ["tab", "l"]:
+            self.tab_offset = (self.tab_offset + 1) % (len(self.tabs))
+            self.show()
+        elif key == "h":
+            self.tab_offset = (self.tab_offset - 1) % (len(self.tabs))
+            self.show()
+        return self._w.keypress(size, key)
+
+    def show(self):
+        headers = []
+        for i in range(len(self.tabs)):
+            txt = self.tabs[i][0]()
+            if i == self.tab_offset:
+                headers.append(self._tab(txt, "heading"))
+            else:
+                headers.append(self._tab(txt, "heading_inactive"))
+        headers = urwid.Columns(headers, dividechars=1)
+        self._w = urwid.Frame(
+            body = self.tabs[self.tab_offset][1](),
+            header = headers
+        )
+        self._w.set_focus("body")

libmproxy/console/window.py

@@ -0,0 +1,72 @@
+import urwid
+from . import signals
+
+
+class Window(urwid.Frame):
+    def __init__(self, master, body, header, footer, helpctx):
+        urwid.Frame.__init__(
+            self,
+            urwid.AttrWrap(body, "background"),
+            header = urwid.AttrWrap(header, "background") if header else None,
+            footer = urwid.AttrWrap(footer, "background") if footer else None
+        )
+        self.master = master
+        self.helpctx = helpctx
+        signals.focus.connect(self.sig_focus)
+
+    def sig_focus(self, sender, section):
+        self.focus_position = section
+
+    def keypress(self, size, k):
+        k = super(self.__class__, self).keypress(size, k)
+        if k == "?":
+            self.master.view_help(self.helpctx)
+        elif k == "c":
+            if not self.master.client_playback:
+                signals.status_prompt_path.send(
+                    self,
+                    prompt = "Client replay",
+                    callback = self.master.client_playback_path
+                )
+            else:
+                signals.status_prompt_onekey.send(
+                    self,
+                    prompt = "Stop current client replay?",
+                    keys = (
+                        ("yes", "y"),
+                        ("no", "n"),
+                    ),
+                    callback = self.master.stop_client_playback_prompt,
+                )
+        elif k == "i":
+            signals.status_prompt.send(
+                self,
+                prompt = "Intercept filter",
+                text = self.master.state.intercept_txt,
+                callback = self.master.set_intercept
+            )
+        elif k == "o":
+            self.master.view_options()
+        elif k == "Q":
+            raise urwid.ExitMainLoop
+        elif k == "q":
+            signals.pop_view_state.send(self)
+        elif k == "S":
+            if not self.master.server_playback:
+                signals.status_prompt_path.send(
+                    self,
+                    prompt = "Server replay path",
+                    callback = self.master.server_playback_path
+                )
+            else:
+                signals.status_prompt_onekey.send(
+                    self,
+                    prompt = "Stop current server replay?",
+                    keys = (
+                        ("yes", "y"),
+                        ("no", "n"),
+                    ),
+                    callback = self.master.stop_server_playback_prompt,
+                )
+        else:
+            return k

libmproxy/controller.py

@@ -1,11 +1,14 @@
 from __future__ import absolute_import
-import Queue, threading
+import Queue
+import threading
+
 
 class DummyReply:
     """
         A reply object that does nothing. Useful when we need an object to seem
         like it has a channel, and during testing.
     """
+
     def __init__(self):
         self.acked = False
 
@@ -19,6 +22,7 @@ class Reply:
         This object is used to respond to the message through the return
         channel.
     """
+
     def __init__(self, obj):
         self.obj = obj
         self.q = Queue.Queue()
@@ -67,11 +71,13 @@ class Slave(threading.Thread):
         Slaves get a channel end-point through which they can send messages to
         the master.
     """
+
     def __init__(self, channel, server):
         self.channel, self.server = channel, server
         self.server.set_channel(channel)
         threading.Thread.__init__(self)
-        self.name = "SlaveThread (%s:%s)" % (self.server.address.host, self.server.address.port)
+        self.name = "SlaveThread (%s:%s)" % (
+            self.server.address.host, self.server.address.port)
 
     def run(self):
         self.server.serve_forever()
@@ -81,6 +87,7 @@ class Master(object):
     """
         Masters get and respond to messages from slaves.
     """
+
     def __init__(self, server):
         """
             server may be None if no server is needed.

libmproxy/dump.py

@@ -53,7 +53,7 @@ class Options(object):
 
 
 def str_response(resp):
-    r = "%s %s"%(resp.code, resp.msg)
+    r = "%s %s" % (resp.code, resp.msg)
     if resp.is_replay:
         r = "[replay] " + r
     return r
@@ -64,7 +64,7 @@ def str_request(f, showhost):
         c = f.client_conn.address.host
     else:
         c = "[replay]"
-    r = "%s %s %s"%(c, f.request.method, f.request.pretty_url(showhost))
+    r = "%s %s %s" % (c, f.request.method, f.request.pretty_url(showhost))
     if f.request.stickycookie:
         r = "[stickycookie] " + r
     return r
@@ -102,7 +102,7 @@ class DumpMaster(flow.FlowMaster):
             try:
                 f = file(path, options.outfile[1])
                 self.start_stream(f, self.filt)
-            except IOError, v:
+            except IOError as v:
                 raise DumpError(v.strerror)
 
         if options.replacements:
@@ -140,7 +140,7 @@ class DumpMaster(flow.FlowMaster):
         if options.rfile:
             try:
                 self.load_flows_file(options.rfile)
-            except flow.FlowReadError, v:
+            except flow.FlowReadError as v:
                 self.add_event("Flow file corrupted.", "error")
                 raise DumpError(v)
 
@@ -171,7 +171,7 @@ class DumpMaster(flow.FlowMaster):
 
     def _print_message(self, message):
         if self.o.flow_detail >= 2:
-            print(self.indent(4, message.headers), file=self.outfile)
+            print(self.indent(4, message.headers.format()), file=self.outfile)
         if self.o.flow_detail >= 3:
             if message.content == http.CONTENT_MISSING:
                 print(self.indent(4, "(content missing)"), file=self.outfile)
@@ -181,12 +181,18 @@ class DumpMaster(flow.FlowMaster):
                 if not utils.isBin(content):
                     try:
                         jsn = json.loads(content)
-                        print(self.indent(4, json.dumps(jsn, indent=2)), file=self.outfile)
+                        print(
+                            self.indent(
+                                4,
+                                json.dumps(
+                                    jsn,
+                                    indent=2)),
+                            file=self.outfile)
                     except ValueError:
                         print(self.indent(4, content), file=self.outfile)
                 else:
                     d = netlib.utils.hexdump(content)
-                    d = "\n".join("%s\t%s %s"%i for i in d)
+                    d = "\n".join("%s\t%s %s" % i for i in d)
                     print(self.indent(4, d), file=self.outfile)
         if self.o.flow_detail >= 2:
             print("", file=self.outfile)
@@ -207,8 +213,13 @@ class DumpMaster(flow.FlowMaster):
             if f.response.content == http.CONTENT_MISSING:
                 sz = "(content missing)"
             else:
-                sz = utils.pretty_size(len(f.response.content))
+                sz = netlib.utils.pretty_size(len(f.response.content))
-            print(" << %s %s" % (str_response(f.response), sz), file=self.outfile)
+            print(
+                " << %s %s" %
+                (str_response(
+                    f.response),
+                    sz),
+                file=self.outfile)
             self._print_message(f.response)
 
         if f.error:

libmproxy/encoding.py

@@ -3,12 +3,14 @@
 """
 from __future__ import absolute_import
 import cStringIO
-import gzip, zlib
+import gzip
+import zlib
 
 __ALL__ = ["ENCODINGS"]
 
 ENCODINGS = set(["identity", "gzip", "deflate"])
 
+
 def decode(e, content):
     encoding_map = {
         "identity": identity,
@@ -19,6 +21,7 @@ def decode(e, content):
         return None
     return encoding_map[e](content)
 
+
 def encode(e, content):
     encoding_map = {
         "identity": identity,
@@ -29,6 +32,7 @@ def encode(e, content):
         return None
     return encoding_map[e](content)
 
+
 def identity(content):
     """
         Returns content unchanged. Identity is the default value of
@@ -36,6 +40,7 @@ def identity(content):
     """
     return content
 
+
 def decode_gzip(content):
     gfile = gzip.GzipFile(fileobj=cStringIO.StringIO(content))
     try:
@@ -43,6 +48,7 @@ def decode_gzip(content):
     except (IOError, EOFError):
         return None
 
+
 def encode_gzip(content):
     s = cStringIO.StringIO()
     gf = gzip.GzipFile(fileobj=s, mode='wb')
@@ -50,6 +56,7 @@ def encode_gzip(content):
     gf.close()
     return s.getvalue()
 
+
 def decode_deflate(content):
     """
         Returns decompressed data for DEFLATE. Some servers may respond with
@@ -67,6 +74,7 @@ def decode_deflate(content):
     except zlib.error:
         return None
 
+
 def encode_deflate(content):
     """
         Returns compressed content, always including zlib header and checksum.

libmproxy/filt.py

@@ -32,16 +32,17 @@
         rex         Equivalent to ~u rex
 """
 from __future__ import absolute_import
-import re, sys
+import re
+import sys
 from .contrib import pyparsing as pp
 from .protocol.http import decoded
 
 
 class _Token:
     def dump(self, indent=0, fp=sys.stdout):
-        print >> fp, "\t"*indent, self.__class__.__name__,
+        print >> fp, "\t" * indent, self.__class__.__name__,
         if hasattr(self, "expr"):
-            print >> fp, "(%s)"%self.expr,
+            print >> fp, "(%s)" % self.expr,
         print >> fp
 
 
@@ -54,6 +55,7 @@ class _Action(_Token):
 class FErr(_Action):
     code = "e"
     help = "Match error"
+
     def __call__(self, f):
         return True if f.error else False
 
@@ -61,6 +63,7 @@ class FErr(_Action):
 class FReq(_Action):
     code = "q"
     help = "Match request with no response"
+
     def __call__(self, f):
         if not f.response:
             return True
@@ -69,6 +72,7 @@ class FReq(_Action):
 class FResp(_Action):
     code = "s"
     help = "Match response"
+
     def __call__(self, f):
         return True if f.response else False
 
@@ -79,7 +83,7 @@ class _Rex(_Action):
         try:
             self.re = re.compile(self.expr)
         except:
-            raise ValueError, "Cannot compile expression."
+            raise ValueError("Cannot compile expression.")
 
 
 def _check_content_type(expr, o):
@@ -100,6 +104,7 @@ class FAsset(_Action):
         "image/.*",
         "application/x-shockwave-flash"
     ]
+
     def __call__(self, f):
         if f.response:
             for i in self.ASSET_TYPES:
@@ -111,6 +116,7 @@ class FAsset(_Action):
 class FContentType(_Rex):
     code = "t"
     help = "Content-type header"
+
     def __call__(self, f):
         if _check_content_type(self.expr, f.request):
             return True
@@ -122,6 +128,7 @@ class FContentType(_Rex):
 class FRequestContentType(_Rex):
     code = "tq"
     help = "Request Content-Type header"
+
     def __call__(self, f):
         return _check_content_type(self.expr, f.request)
 
@@ -129,6 +136,7 @@ class FRequestContentType(_Rex):
 class FResponseContentType(_Rex):
     code = "ts"
     help = "Response Content-Type header"
+
     def __call__(self, f):
         if f.response:
             return _check_content_type(self.expr, f.response)
@@ -138,6 +146,7 @@ class FResponseContentType(_Rex):
 class FHead(_Rex):
     code = "h"
     help = "Header"
+
     def __call__(self, f):
         if f.request.headers.match_re(self.expr):
             return True
@@ -149,6 +158,7 @@ class FHead(_Rex):
 class FHeadRequest(_Rex):
     code = "hq"
     help = "Request header"
+
     def __call__(self, f):
         if f.request.headers.match_re(self.expr):
             return True
@@ -157,6 +167,7 @@ class FHeadRequest(_Rex):
 class FHeadResponse(_Rex):
     code = "hs"
     help = "Response header"
+
     def __call__(self, f):
         if f.response and f.response.headers.match_re(self.expr):
             return True
@@ -165,6 +176,7 @@ class FHeadResponse(_Rex):
 class FBod(_Rex):
     code = "b"
     help = "Body"
+
     def __call__(self, f):
         if f.request and f.request.content:
             with decoded(f.request):
@@ -180,6 +192,7 @@ class FBod(_Rex):
 class FBodRequest(_Rex):
     code = "bq"
     help = "Request body"
+
     def __call__(self, f):
         if f.request and f.request.content:
             with decoded(f.request):
@@ -190,6 +203,7 @@ class FBodRequest(_Rex):
 class FBodResponse(_Rex):
     code = "bs"
     help = "Response body"
+
     def __call__(self, f):
         if f.response and f.response.content:
             with decoded(f.response):
@@ -200,6 +214,7 @@ class FBodResponse(_Rex):
 class FMethod(_Rex):
     code = "m"
     help = "Method"
+
     def __call__(self, f):
         return bool(re.search(self.expr, f.request.method, re.IGNORECASE))
 
@@ -207,6 +222,7 @@ class FMethod(_Rex):
 class FDomain(_Rex):
     code = "d"
     help = "Domain"
+
     def __call__(self, f):
         return bool(re.search(self.expr, f.request.host, re.IGNORECASE))
 
@@ -215,6 +231,7 @@ class FUrl(_Rex):
     code = "u"
     help = "URL"
     # FUrl is special, because it can be "naked".
+
     @classmethod
     def make(klass, s, loc, toks):
         if len(toks) > 1:
@@ -233,6 +250,7 @@ class _Int(_Action):
 class FCode(_Int):
     code = "c"
     help = "HTTP response code"
+
     def __call__(self, f):
         if f.response and f.response.code == self.num:
             return True
@@ -243,9 +261,9 @@ class FAnd(_Token):
         self.lst = lst
 
     def dump(self, indent=0, fp=sys.stdout):
-        print >> fp, "\t"*indent, self.__class__.__name__
+        print >> fp, "\t" * indent, self.__class__.__name__
         for i in self.lst:
-            i.dump(indent+1, fp)
+            i.dump(indent + 1, fp)
 
     def __call__(self, f):
         return all(i(f) for i in self.lst)
@@ -256,9 +274,9 @@ class FOr(_Token):
         self.lst = lst
 
     def dump(self, indent=0, fp=sys.stdout):
-        print >> fp, "\t"*indent, self.__class__.__name__
+        print >> fp, "\t" * indent, self.__class__.__name__
         for i in self.lst:
-            i.dump(indent+1, fp)
+            i.dump(indent + 1, fp)
 
     def __call__(self, f):
         return any(i(f) for i in self.lst)
@@ -269,7 +287,7 @@ class FNot(_Token):
         self.itm = itm[0]
 
     def dump(self, indent=0, fp=sys.stdout):
-        print >> fp, "\t"*indent, self.__class__.__name__
+        print >> fp, "\t" * indent, self.__class__.__name__
         self.itm.dump(indent + 1, fp)
 
     def __call__(self, f):
@@ -299,26 +317,28 @@ filt_rex = [
 filt_int = [
     FCode
 ]
+
+
 def _make():
     # Order is important - multi-char expressions need to come before narrow
     # ones.
     parts = []
     for klass in filt_unary:
-        f = pp.Literal("~%s"%klass.code)
+        f = pp.Literal("~%s" % klass.code)
         f.setParseAction(klass.make)
         parts.append(f)
 
-    simplerex = "".join(c for c in pp.printables if c not in  "()~'\"")
+    simplerex = "".join(c for c in pp.printables if c not in "()~'\"")
     rex = pp.Word(simplerex) |\
-          pp.QuotedString("\"", escChar='\\') |\
+        pp.QuotedString("\"", escChar='\\') |\
-          pp.QuotedString("'", escChar='\\')
+        pp.QuotedString("'", escChar='\\')
     for klass in filt_rex:
-        f = pp.Literal("~%s"%klass.code) + rex.copy()
+        f = pp.Literal("~%s" % klass.code) + rex.copy()
         f.setParseAction(klass.make)
         parts.append(f)
 
     for klass in filt_int:
-        f = pp.Literal("~%s"%klass.code) + pp.Word(pp.nums)
+        f = pp.Literal("~%s" % klass.code) + pp.Word(pp.nums)
         f.setParseAction(klass.make)
         parts.append(f)
 
@@ -328,14 +348,20 @@ def _make():
     parts.append(f)
 
     atom = pp.MatchFirst(parts)
-    expr = pp.operatorPrecedence(
+    expr = pp.operatorPrecedence(atom,
-                atom,
+                                 [(pp.Literal("!").suppress(),
-                [
+                                   1,
-                    (pp.Literal("!").suppress(), 1, pp.opAssoc.RIGHT, lambda x: FNot(*x)),
+                                   pp.opAssoc.RIGHT,
-                    (pp.Literal("&").suppress(), 2, pp.opAssoc.LEFT, lambda x: FAnd(*x)),
+                                   lambda x: FNot(*x)),
-                    (pp.Literal("|").suppress(), 2, pp.opAssoc.LEFT, lambda x: FOr(*x)),
+                                     (pp.Literal("&").suppress(),
-                ]
+                                      2,
-           )
+                                      pp.opAssoc.LEFT,
+                                      lambda x: FAnd(*x)),
+                                     (pp.Literal("|").suppress(),
+                                      2,
+                                      pp.opAssoc.LEFT,
+                                      lambda x: FOr(*x)),
+                                  ])
     expr = pp.OneOrMore(expr)
     return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
 bnf = _make()
@@ -355,15 +381,15 @@ def parse(s):
 help = []
 for i in filt_unary:
     help.append(
-        ("~%s"%i.code, i.help)
+        ("~%s" % i.code, i.help)
     )
 for i in filt_rex:
     help.append(
-        ("~%s regex"%i.code, i.help)
+        ("~%s regex" % i.code, i.help)
     )
 for i in filt_int:
     help.append(
-        ("~%s int"%i.code, i.help)
+        ("~%s int" % i.code, i.help)
     )
 help.sort()
 help.extend(
@@ -373,4 +399,4 @@ help.extend(
         ("|", "or"),
         ("(...)", "grouping"),
     ]
-)
+)

libmproxy/flow.py

@@ -17,9 +17,6 @@ from .proxy.config import HostMatcher
 from .proxy.connection import ClientConnection, ServerConnection
 import urlparse
 
-ODict = odict.ODict
-ODictCaseless = odict.ODictCaseless
-
 
 class AppRegistry:
     def __init__(self):
@@ -165,7 +162,8 @@ class StreamLargeBodies(object):
             r.headers, is_request, flow.request.method, code
         )
         if not (0 <= expected_size <= self.max_size):
-            r.stream = r.stream or True  # r.stream may already be a callable, which we want to preserve.
+            # r.stream may already be a callable, which we want to preserve.
+            r.stream = r.stream or True
 
 
 class ClientPlaybackState:
@@ -203,8 +201,16 @@ class ClientPlaybackState:
 
 
 class ServerPlaybackState:
-    def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content,
+    def __init__(
-                 ignore_payload_params, ignore_host):
+            self,
+            headers,
+            flows,
+            exit,
+            nopop,
+            ignore_params,
+            ignore_content,
+            ignore_payload_params,
+            ignore_host):
         """
             headers: Case-insensitive list of request headers that should be
             included in request-response matching.
@@ -232,7 +238,7 @@ class ServerPlaybackState:
         r = flow.request
 
         _, _, path, _, query, _ = urlparse.urlparse(r.url)
-        queriesArray = urlparse.parse_qsl(query)
+        queriesArray = urlparse.parse_qsl(query, keep_blank_values=True)
 
         key = [
             str(r.port),
@@ -242,7 +248,7 @@ class ServerPlaybackState:
         ]
 
         if not self.ignore_content:
-            form_contents = r.get_form_urlencoded()
+            form_contents = r.get_form()
             if self.ignore_payload_params and form_contents:
                 key.extend(
                     p for p in form_contents
@@ -271,7 +277,7 @@ class ServerPlaybackState:
                 # to prevent a mismatch between unicode/non-unicode.
                 v = [str(x) for x in v]
                 hdrs.append((i, v))
-            key.append(repr(hdrs))
+            key.append(hdrs)
         return hashlib.sha256(repr(key)).digest()
 
     def next_flow(self, request):
@@ -462,8 +468,9 @@ class FlowStore(FlowList):
         Notifies the state that a flow has been updated.
         The flow must be present in the state.
         """
-        for view in self.views:
+        if f in self:
-            view._update(f)
+            for view in self.views:
+                view._update(f)
 
     def _remove(self, f):
         """
@@ -534,7 +541,8 @@ class State(object):
     def flow_count(self):
         return len(self.flows)
 
-    # TODO: All functions regarding flows that don't cause side-effects should be moved into FlowStore.
+    # TODO: All functions regarding flows that don't cause side-effects should
+    # be moved into FlowStore.
     def index(self, f):
         return self.flows.index(f)
 
@@ -593,6 +601,10 @@ class State(object):
     def accept_all(self, master):
         self.flows.accept_all(master)
 
+    def backup(self, f):
+        f.backup()
+        self.update_flow(f)
+
     def revert(self, f):
         f.revert()
         self.update_flow(f)
@@ -658,7 +670,7 @@ class FlowMaster(controller.Master):
         """
         try:
             s = script.Script(command, self)
-        except script.ScriptError, v:
+        except script.ScriptError as v:
             return v.args[0]
         self.scripts.append(s)
 
@@ -722,8 +734,17 @@ class FlowMaster(controller.Master):
     def stop_client_playback(self):
         self.client_playback = None
 
-    def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params,
+    def start_server_playback(
-                              ignore_content, ignore_payload_params, ignore_host):
+            self,
+            flows,
+            kill,
+            headers,
+            exit,
+            nopop,
+            ignore_params,
+            ignore_content,
+            ignore_payload_params,
+            ignore_host):
         """
             flows: List of flows.
             kill: Boolean, should we kill requests not part of the replay?
@@ -732,9 +753,15 @@ class FlowMaster(controller.Master):
             ignore_payload_params: list of content params to ignore in server replay
             ignore_host: true if request host should be ignored in server replay
         """
-        self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, 
+        self.server_playback = ServerPlaybackState(
-                                                   ignore_params, ignore_content,
+            headers,
-                                                   ignore_payload_params, ignore_host)
+            flows,
+            exit,
+            nopop,
+            ignore_params,
+            ignore_content,
+            ignore_payload_params,
+            ignore_host)
         self.kill_nonreplay = kill
 
     def stop_server_playback(self):
@@ -771,6 +798,8 @@ class FlowMaster(controller.Master):
             if all(e):
                 self.shutdown()
             self.client_playback.tick(self)
+            if self.client_playback.done():
+                self.client_playback = None
 
         return super(FlowMaster, self).tick(q, timeout)
 
@@ -780,25 +809,38 @@ class FlowMaster(controller.Master):
     def create_request(self, method, scheme, host, port, path):
         """
             this method creates a new artificial and minimalist request also adds it to flowlist
-        """        
+        """
         c = ClientConnection.from_state(dict(
-                address=dict(address=(host, port), use_ipv6=False),
+            address=dict(address=(host, port), use_ipv6=False),
-                clientcert=None
+            clientcert=None
-            ))
+        ))
 
         s = ServerConnection.from_state(dict(
-                address=dict(address=(host, port), use_ipv6=False),
+            address=dict(address=(host, port), use_ipv6=False),
-                state=[],
+            state=[],
-                source_address=None, #source_address=dict(address=(host, port), use_ipv6=False),
+            source_address=None,
-                cert=None,
+            # source_address=dict(address=(host, port), use_ipv6=False),
-                sni=host,
+            cert=None,
-                ssl_established=True
+            sni=host,
-            ))
+            ssl_established=True
-        f = http.HTTPFlow(c,s);
+        ))
+        f = http.HTTPFlow(c, s)
         headers = ODictCaseless()
-        
+
-        req = http.HTTPRequest("absolute", method, scheme, host, port, path, (1, 1), headers, None,
+        req = http.HTTPRequest(
-                                 None, None, None)
+            "absolute",
+            method,
+            scheme,
+            host,
+            port,
+            path,
+            (1,
+             1),
+            headers,
+            None,
+            None,
+            None,
+            None)
         f.request = req
         return self.load_flow(f)
 
@@ -809,7 +851,8 @@ class FlowMaster(controller.Master):
 
         if self.server and self.server.config.mode == "reverse":
             f.request.host, f.request.port = self.server.config.mode.dst[2:]
-            f.request.scheme = "https" if self.server.config.mode.dst[1] else "http"
+            f.request.scheme = "https" if self.server.config.mode.dst[
+                1] else "http"
 
         f.reply = controller.DummyReply()
         if f.request:
@@ -836,7 +879,7 @@ class FlowMaster(controller.Master):
         try:
             f = file(path, "rb")
             freader = FlowReader(f)
-        except IOError, v:
+        except IOError as v:
             raise FlowReadError(v.strerror)
         return self.load_flows(freader)
 
@@ -877,7 +920,8 @@ class FlowMaster(controller.Master):
             f.backup()
             f.request.is_replay = True
             if f.request.content:
-                f.request.headers["Content-Length"] = [str(len(f.request.content))]
+                f.request.headers[
+                    "Content-Length"] = [str(len(f.request.content))]
             f.response = None
             f.error = None
             self.process_new_request(f)
@@ -1028,7 +1072,7 @@ class FlowReader:
         """
         off = 0
         try:
-            while 1:
+            while True:
                 data = tnetstring.load(self.fo)
                 if tuple(data["version"][:2]) != version.IVERSION[:2]:
                     v = ".".join(str(i) for i in data["version"])
@@ -1037,7 +1081,7 @@ class FlowReader:
                     )
                 off = self.fo.tell()
                 yield handle.protocols[data["type"]]["flow"].from_state(data)
-        except ValueError, v:
+        except ValueError as v:
             # Error is due to EOF
             if self.fo.tell() == off and self.fo.read() == '':
                 return

libmproxy/main.py

@@ -70,27 +70,29 @@ def get_server(dummy_server, options):
     else:
         try:
             return ProxyServer(options)
-        except ProxyServerError, v:
+        except ProxyServerError as v:
             print(str(v), file=sys.stderr)
             sys.exit(1)
 
 
-def mitmproxy():  # pragma: nocover
+def mitmproxy(args=None):  # pragma: nocover
     from . import console
 
     check_versions()
     assert_utf8_env()
 
     parser = cmdline.mitmproxy()
-    options = parser.parse_args()
+    options = parser.parse_args(args)
     if options.quiet:
         options.verbose = 0
 
     proxy_config = process_proxy_options(parser, options)
     console_options = console.Options(**cmdline.get_common_options(options))
     console_options.palette = options.palette
+    console_options.palette_transparent = options.palette_transparent
     console_options.eventlog = options.eventlog
     console_options.intercept = options.intercept
+    console_options.limit = options.limit
 
     server = get_server(console_options.no_server, proxy_config)
 
@@ -101,13 +103,13 @@ def mitmproxy():  # pragma: nocover
         pass
 
 
-def mitmdump():  # pragma: nocover
+def mitmdump(args=None):  # pragma: nocover
     from . import dump
 
     check_versions()
 
     parser = cmdline.mitmdump()
-    options = parser.parse_args()
+    options = parser.parse_args(args)
     if options.quiet:
         options.verbose = 0
         options.flow_detail = 0
@@ -135,13 +137,13 @@ def mitmdump():  # pragma: nocover
         pass
 
 
-def mitmweb():  # pragma: nocover
+def mitmweb(args=None):  # pragma: nocover
     from . import web
 
     check_versions()
     parser = cmdline.mitmweb()
 
-    options = parser.parse_args()
+    options = parser.parse_args(args)
     if options.quiet:
         options.verbose = 0
 

libmproxy/onboarding/app.py

@@ -45,7 +45,10 @@ class PEM(tornado.web.RequestHandler):
     def get(self):
         p = os.path.join(self.request.master.server.config.cadir, self.filename)
         self.set_header("Content-Type", "application/x-x509-ca-cert")
-        self.set_header("Content-Disposition", "inline; filename={}".format(self.filename))
+        self.set_header(
+            "Content-Disposition",
+            "inline; filename={}".format(
+                self.filename))
 
         with open(p, "rb") as f:
             self.write(f.read())
@@ -59,7 +62,10 @@ class P12(tornado.web.RequestHandler):
     def get(self):
         p = os.path.join(self.request.master.server.config.cadir, self.filename)
         self.set_header("Content-Type", "application/x-pkcs12")
-        self.set_header("Content-Disposition", "inline; filename={}".format(self.filename))
+        self.set_header(
+            "Content-Disposition",
+            "inline; filename={}".format(
+                self.filename))
 
         with open(p, "rb") as f:
             self.write(f.read())
@@ -78,7 +84,6 @@ application = tornado.web.Application(
             }
         ),
     ],
-    #debug=True
+    # debug=True
 )
 mapp = Adapter(application)
-

libmproxy/platform/__init__.py

@@ -8,9 +8,9 @@ if sys.platform == "linux2":
 elif sys.platform == "darwin":
     from . import osx
     resolver = osx.Resolver
-elif sys.platform == "freebsd10":
+elif sys.platform.startswith("freebsd"):
     from . import osx
     resolver = osx.Resolver
 elif sys.platform == "win32":
     from . import windows
-    resolver = windows.Resolver
+    resolver = windows.Resolver

libmproxy/platform/linux.py

@@ -1,4 +1,5 @@
-import socket, struct
+import socket
+import struct
 
 # Python socket module does not have this constant
 SO_ORIGINAL_DST = 80

libmproxy/platform/osx.py

@@ -21,6 +21,7 @@ class Resolver(object):
         peer = csock.getpeername()
         stxt = subprocess.check_output(self.STATECMD, stderr=subprocess.STDOUT)
         if "sudo: a password is required" in stxt:
-            raise RuntimeError("Insufficient privileges to access pfctl. "
+            raise RuntimeError(
-                               "See http://mitmproxy.org/doc/transparent/osx.html for details.")
+                "Insufficient privileges to access pfctl. "
+                "See http://mitmproxy.org/doc/transparent/osx.html for details.")
         return pf.lookup(peer[0], peer[1], stxt)

libmproxy/platform/pf.py

@@ -13,7 +13,7 @@ def lookup(address, port, s):
         if "ESTABLISHED:ESTABLISHED" in i and spec in i:
             s = i.split()
             if len(s) > 4:
-                if sys.platform == "freebsd10":
+                if sys.platform.startswith("freebsd"):
                     # strip parentheses for FreeBSD pfctl
                     s = s[3][1:-1].split(":")
                 else:
@@ -21,4 +21,4 @@ def lookup(address, port, s):
 
                 if len(s) == 2:
                     return s[0], int(s[1])
-    raise RuntimeError("Could not resolve original destination.")
+    raise RuntimeError("Could not resolve original destination.")