document --ignore, fix #350

doc-src/_nav.html

@@ -10,18 +10,21 @@
 
     <li class="nav-header">Features</li>
         $!nav("anticache.html", this, state)!$
-        $!nav("clientreplay.html", this, state)!$
+
         $!nav("filters.html", this, state)!$
-        $!nav("upstreamproxy.html", this, state)!$
-        $!nav("proxyauth.html", this, state)!$
         $!nav("replacements.html", this, state)!$
+        $!nav("clientreplay.html", this, state)!$
         $!nav("serverreplay.html", this, state)!$
         $!nav("setheaders.html", this, state)!$
+        $!nav("passthrough.html", this, state)!$
         $!nav("sticky.html", this, state)!$
         $!nav("reverseproxy.html", this, state)!$
+        $!nav("upstreamproxy.html", this, state)!$
         $!nav("upstreamcerts.html", this, state)!$
+        $!nav("proxyauth.html", this, state)!$
         $!nav("responsestreaming.html", this, state)!$
 
+
     <li class="nav-header">Installing Certificates</li>
         $!nav("ssl.html", this, state)!$
         $!nav("certinstall/webapp.html", this, state)!$

doc-src/features/index.py

@@ -4,13 +4,14 @@ pages = [
     Page("anticache.html", "Anticache"),
     Page("clientreplay.html", "Client-side replay"),
     Page("filters.html", "Filter expressions"),
-    Page("upstreamproxy.html", "Upstream proxy mode"),
+    Page("passthrough.html", "Ignore Domains"),
+    Page("proxyauth.html", "Proxy Authentication"),
+    Page("replacements.html", "Replacements"),
+    Page("responsestreaming.html", "Response Streaming"),
+    Page("reverseproxy.html", "Reverse proxy mode"),
     Page("setheaders.html", "Set Headers"),
     Page("serverreplay.html", "Server-side replay"),
     Page("sticky.html", "Sticky cookies and auth"),
-    Page("proxyauth.html", "Proxy Authentication"),
-    Page("replacements.html", "Replacements"),
-    Page("reverseproxy.html", "Reverse proxy mode"),
     Page("upstreamcerts.html", "Upstream Certs"),
-    Page("responsestreaming.html", "Response Streaming"),
+    Page("upstreamproxy.html", "Upstream proxy mode"),
 ]

doc-src/features/passthrough.html

@@ -0,0 +1,77 @@
+There are a couple of reasons why you may want to exempt some traffic from mitmproxy's interception mechanism:
+
+- **Certificate pinning:** Some traffic is is protected using
+  [certificate pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's
+  interception leads to errors. For example, Windows Update or the Apple App Store fail to work if mitmproxy is active.
+- **Non-HTTP traffic:** WebSockets or other non-http protocols are not supported by mitmproxy yet. You can exempt the
+  domain from processing, which would otherwise fail.
+- **Convenience:** You really don't care about some parts of the traffic and just want them to go away.
+
+If you want to ignore traffic from mitmproxy's processing because of large response bodies, check out the
+[response streaming](@!urlTo("responsestreaming.html")!@) feature.
+
+## How it works
+
+
+<table class="table">
+    <tbody>
+        <tr>
+            <th width="20%">command-line</th> <td>--ignore regex</td>
+        </tr>
+        <tr>
+            <th>mitmproxy shortcut</th> <td><b>I</b></td>
+        </tr>
+    </tbody>
+</table>
+
+
+mitmproxy allows you to specify a regex which is matched against a <code>host:port</code> string (e.g. "example.com:443")
+to determine hosts that should be excluded. 
+
+There are two important quirks to consider:
+
+- **In transparent mode, the ignore pattern is matched against the IP.** While we usually infer the hostname from the 
+  Host header if the --host argument is passed to mitmproxy, we do not have access to this information before the SSL
+  handshake.
+- In regular mode, explicit HTTP requests are never ignored.[^explicithttp] The ignore pattern is applied on CONNECT
+  requests, which initiate HTTPS or clear-text WebSocket connections.
+
+
+### Tutorial
+
+If you just want to ignore one specific domain, there's usually a bulletproof method to do so:
+
+1. Run mitmproxy or mitmdump in verbose mode (-v) and observe the host:port information in the serverconnect 
+messages. mitmproxy will filter on these.
+2. Take the host:port string, surround it with ^ and $, escape all dots (. becomes \\.) 
+and use this as your ignore pattern:
+
+<pre class="terminal">
+$ mitmdump -v
+127.0.0.1:50588: clientconnect
+127.0.0.1:50588: request
+  -> CONNECT example.com:443 HTTP/1.1
+127.0.0.1:50588: Set new server address: example.com:443
+<span style="color: white">127.0.0.1:50588: serverconnect
+  -> example.com:443</span>
+^C
+$ <span style="color: white">mitmproxy --ignore ^example\.com:443$</span>    
+</pre>
+
+Here are some other examples for ignore patterns:
+<pre>
+# Exempt traffic from the iOS App Store (usually just works):
+--ignore apple.com:443  
+# "Correct" version without false-positives:
+--ignore ^(.+\.)?apple\.com:443$ 
+    
+# Ignore example.com on all ports, but no subdomains:
+--ignore ^example.com:
+    
+# Transparent mode:
+--ignore 17\.178\.96\.59:443
+# IP address range:
+--ignore 17\.178\.\d+\.\d+:443
+</pre>
+
+[^explicithttp]: This stems from an limitation of explicit HTTP proxying: A single connection can be re-used for multiple target domains - a <code>GET http://example.com/</code> request may be followed by a <code>GET http://evil.com/</code> request on the same connection. If we start to ignore the connection after the first request, we would miss the relevant second one.

libmproxy/console/help.py

@@ -36,6 +36,7 @@ class HelpView(urwid.ListBox):
         keys = [
             ("c", "client replay"),
             ("H", "edit global header set patterns"),
+            ("I", "set ignore pattern"),
             ("i", "set interception pattern"),
             ("M", "change global default display mode"),
                 (None,
@@ -115,7 +116,7 @@ class HelpView(urwid.ListBox):
             ("q", "quit / return to flow list"),
             ("Q", "quit without confirm prompt"),
             ("R", "edit replacement patterns"),
-            ("s", "set/unset script"),
+            ("s", "add/remove scripts"),
             ("S", "server replay"),
             ("t", "set sticky cookie expression"),
             ("u", "set sticky auth expression"),