Merge branch 'master' of ssh.github.com:cortesi/mitmproxy

doc-src/02-docstyle.css

@@ -10,3 +10,7 @@ body {
 .nowrap {
      white-space: nowrap;
 }
+
+h1 {
+    line-height: 1.1;
+}

doc-src/_layout.html

@@ -6,7 +6,7 @@
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
       </a>
-      <a class="brand" href="@!urlTo(idxpath)!@">mitmproxy 0.9 docs</a>
+      <a class="brand" href="@!urlTo(idxpath)!@">mitmproxy $!VERSION!$ docs</a>
       </div><!--/.nav-collapse -->
     </div>
   </div>
@@ -51,9 +51,10 @@
                 $!nav("transparent/linux.html", this, state)!$
                 $!nav("transparent/osx.html", this, state)!$
 
-             <li class="nav-header">Tutorials</li>
+            <li class="nav-header">Tutorials</li>
                 $!nav("tutorials/30second.html", this, state)!$
                 $!nav("tutorials/gamecenter.html", this, state)!$
+                $!nav("tutorials/transparent-dhcp.html", this, state)!$
 
             <li class="nav-header">Scripting mitmproxy</li>
                 $!nav("scripting/inlinescripts.html", this, state)!$

doc-src/_websitelayout.html

@@ -6,7 +6,7 @@
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
       </a>
-      <a class="brand" href="@!urlTo("/index.html")!@">mitmproxy</a>
+      <a class="brand" href="@!urlTo('/index.html')!@">mitmproxy</a>
       <div class="nav">
         <ul class="nav">
           <li $!'class="active"' if this.match("/index.html", True) else ""!$> <a href="@!top!@/index.html">home</a> </li>

doc-src/index.py

@@ -1,11 +1,12 @@
-import os, sys
+import os, sys, datetime
 import countershape
 from countershape import Page, Directory, PythonModule, markup, model
 import countershape.template
 sys.path.insert(0, "..")
-from libmproxy import filt
+from libmproxy import filt, version
 
-MITMPROXY_SRC = "~/mitmproxy/mitmproxy"
+MITMPROXY_SRC = os.path.abspath("..")
+ns.VERSION = version.VERSION
 
 if ns.options.website:
     ns.idxpath = "doc/index.html"
@@ -16,18 +17,20 @@ else:
 
 
 ns.title = countershape.template.Template(None, "<h1>@!this.title!@</h1>")
-this.titlePrefix = "mitmproxy 0.10 - "
+this.titlePrefix = "%s - " % version.NAMEVERSION
 this.markup = markup.Markdown(extras=["footnotes"])
 
 ns.docMaintainer = "Aldo Cortesi"
 ns.docMaintainerEmail = "aldo@corte.si"
-ns.copyright = u"\u00a9 mitmproxy project, 2013"
+ns.copyright = u"\u00a9 mitmproxy project, %s" % datetime.date.today().year
 
 def mpath(p):
     p = os.path.join(MITMPROXY_SRC, p)
     return os.path.expanduser(p)
 
-ns.index_contents = file(mpath("README.mkd")).read()
+with open(mpath("README.mkd")) as f:
+        readme = f.read()
+        ns.index_contents = readme.split("\n", 1)[1] #remove first line (contains build status)
 
 def example(s):
     d = file(mpath(s)).read().rstrip()

doc-src/mitmproxy.html

@@ -9,7 +9,7 @@ documentation from any __mitmproxy__ screen.
 
 The flow list shows an index of captured flows in chronological order. 
 
-<img src="@!urlTo("screenshots/mitmproxy.png")!@"/>
+<img src="@!urlTo('screenshots/mitmproxy.png')!@"/>
 
 - __1__: A GET request, returning a 302 Redirect response.
 - __2__: A GET request, returning 16.75kb of text/html data.
@@ -32,7 +32,7 @@ interfaces.
 
 The __Flow View__ lets you inspect and manipulate a single flow:
 
-<img src="@!urlTo("screenshots/mitmproxy-flowview.png")!@"/>
+<img src="@!urlTo('screenshots/mitmproxy-flowview.png')!@"/>
 
 - __1__: Flow summary.
 - __2__: The Request/Response tabs, showing you which part of the flow you are
@@ -65,13 +65,13 @@ At the moment, the Grid Editor is used in four parts of mitmproxy:
 If there is is no data, an empty editor will be started to let you add some.
 Here is the editor showing the headers from a request:
 
-<img src="@!urlTo("screenshots/mitmproxy-kveditor.png")!@"/>
+<img src="@!urlTo('screenshots/mitmproxy-kveditor.png')!@"/>
 
 To edit, navigate to the key or value you want to modify using the arrow or vi
 navigation keys, and press enter. The background color will change to show that
 you are in edit mode for the specified field:
 
-<img src="@!urlTo("screenshots/mitmproxy-kveditor-editmode.png")!@"/>
+<img src="@!urlTo('screenshots/mitmproxy-kveditor-editmode.png')!@"/>
 
 Modify the field as desired, then press escape to exit edit mode when you're
 done. You can also add a row (_a_ key), delete a row (_d_ key), spawn an

doc-src/transparent/linux.html

@@ -3,7 +3,7 @@ achieve transparent mode.
 
 <ol class="tlist">
 
-    <li> <a href="@!urlTo("ssl.html")!@">Install the mitmproxy
+    <li> <a href="@!urlTo('ssl.html')!@">Install the mitmproxy
     certificates on the test device</a>. </li>
 
     <li> Enable IP forwarding:
@@ -38,3 +38,6 @@ iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8
     running as the default gateway.</li>
 
 </ol>
+
+
+For a detailed walkthrough, have a look at the <a href="@!urlTo('tutorials/transparent-dhcp.html')!@"><i>Transparently proxify virtual machines</i></a> tutorial.

doc-src/transparent/osx.html

@@ -7,7 +7,7 @@ OSX.
 
 <ol class="tlist">
 
-    <li> <a href="@!urlTo("ssl.html")!@">Install the mitmproxy
+    <li> <a href="@!urlTo('ssl.html')!@">Install the mitmproxy
     certificates on the test device</a>. </li>
 
     <li> Enable IP forwarding:

doc-src/tutorials/gamecenter.html

@@ -17,14 +17,14 @@ Worm](http://itunes.apple.com/us/app/super-mega-worm/id388541990?mt=8) - a
 great little retro-apocalyptic sidescroller for the iPhone: 
 
 <center>
-    <img src="@!urlTo("tutorials/supermega.png")!@"/>
+    <img src="@!urlTo('tutorials/supermega.png')!@"/>
 </center>
 
 After finishing a game (take your time), watch the traffic flowing through
 mitmproxy:
 
 <center>
-    <img src="@!urlTo("tutorials/one.png")!@"/>
+    <img src="@!urlTo('tutorials/one.png')!@"/>
 </center>
 
 We see a bunch of things we might expect - initialisation, the retrieval of
@@ -99,7 +99,7 @@ replay.
 ## The glorious result and some intrigue
 
 <center>
-    <img src="@!urlTo("tutorials/leaderboard.png")!@"/>
+    <img src="@!urlTo('tutorials/leaderboard.png')!@"/>
 </center>
 
 And that's it - according to the records, I am the greatest Super Mega Worm

doc-src/tutorials/index.py

@@ -3,4 +3,5 @@ from countershape import Page
 pages = [
     Page("30second.html", "Client playback: a 30 second example"),
     Page("gamecenter.html", "Setting highscores on Apple's GameCenter"),
-]
+    Page("transparent-dhcp.html", "Transparently proxify virtual machines")
+]

doc-src/tutorials/transparent-dhcp.html

@@ -0,0 +1,54 @@
+This walkthrough illustrates how to set up transparent proxying with mitmproxy. We use VirtualBox VMs with an Ubuntu proxy machine in this example, but the general principle can be applied to other setups.
+
+1.  **Configure VirtualBox Network Adapters for the proxy machine**  
+    The network setup is simple:  `internet <--> proxy vm <--> (virtual) internal network`.  
+    For the proxy machine, *eth0* represents the outgoing network. *eth1* is connected to the internal network that will be proxified, using a static ip (192.168.3.1).  
+    <hr>VirtualBox configuration:
+    <img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth0.png')!@"/><br><br>
+    <img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth1.png')!@"/>  
+    <br>Proxy VM:
+    <img src="@!urlTo('tutorials/transparent-dhcp/step1_proxy.png')!@"/>  
+    <hr>
+2.  **Configure DHCP and DNS**  
+    We use dnsmasq to provide DHCP and DNS in our internal network. 
+    Dnsmasq is a lightweight server designed to provide DNS (and optionally DHCP and TFTP) services to a small-scale 
+    network.  
+
+    -   Before we get to that, we need to fix some Ubuntu quirks: 
+        **Ubuntu >12.04** runs an internal dnsmasq instance (listening on loopback only) by default 
+        <a href="https://www.stgraber.org/2012/02/24/dns-in-ubuntu-12-04/">[1]</a>. For our use case, this needs to be
+        disabled by changing <br>`dns=dnsmasq` to `#dns=dnsmasq` in */etc/NetworkManager/NetworkManager.conf*  
+        and running `sudo restart network-manager` afterwards.
+    -   Now, dnsmasq can be be installed and configured:  
+        `sudo apt-get install dnsmasq`  
+        Replace */etc/dnsmasq.conf* with the following configuration:  
+        <pre>\# Listen for DNS requests on the internal network
+        interface=eth1
+        \# Act as a DHCP server, assign IP addresses to clients
+        dhcp-range=192.168.3.10,192.168.3.100,96h
+        \# Broadcast gateway and dns server information
+        dhcp-option=option:router,192.168.3.1
+        dhcp-option=option:dns-server,192.168.3.1
+        </pre>
+        Apply changes:  
+        `sudo service dnsmasq restart`
+        <hr>
+        Your proxied machine's network settings should now look similar to this:
+        <img src="@!urlTo('tutorials/transparent-dhcp/step2_proxied_vm.png')!@"/>
+        <hr>
+
+3.  **Set up traffic redirection to mitmproxy**  
+    To redirect traffic to mitmproxy, we need to add two iptables rules:  
+    <pre class="terminal">
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 \
+        -j REDIRECT --to-port 8080
+    iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 \
+        -j REDIRECT --to-port 8080
+    </pre>
+
+4.  If required, <a href="@!urlTo('ssl.html')!@">install the mitmproxy
+    certificates on the test device</a>.
+
+5.  Finally, we can run <code>mitmproxy -T</code>.  
+    The proxied machine cannot to leak any data outside of HTTP or DNS requests.
+

libmproxy/proxy.py

@@ -106,18 +106,19 @@ class RequestReplayThread(threading.Thread):
 
 
 class HandleSNI:
-    def __init__(self, handler, client_conn, host, port, cert, key):
+    def __init__(self, handler, client_conn, host, port, key):
         self.handler, self.client_conn, self.host, self.port = handler, client_conn, host, port
-        self.cert, self.key = cert, key
+        self.key = key
 
     def __call__(self, client_connection):
         try:
             sn = client_connection.get_servername()
             if sn:
                 self.handler.get_server_connection(self.client_conn, "https", self.host, self.port, sn)
+                dummycert = self.handler.find_cert(self.client_conn, self.host, self.port, sn)
                 new_context = SSL.Context(SSL.TLSv1_METHOD)
                 new_context.use_privatekey_file(self.key)
-                new_context.use_certificate(self.cert.x509)
+                new_context.use_certificate(dummycert.x509)
                 client_connection.set_context(new_context)
                 self.handler.sni = sn.decode("utf8").encode("idna")
         # An unhandled exception in this method will core dump PyOpenSSL, so
@@ -331,8 +332,7 @@ class ProxyHandler(tcp.BaseHandler):
     def establish_ssl(self, client_conn, host, port):
         dummycert = self.find_cert(client_conn, host, port, host)
         sni = HandleSNI(
-            self, client_conn, host, port,
-            dummycert, self.config.certfile or self.config.cacert
+            self, client_conn, host, port, self.config.certfile or self.config.cacert
         )
         try:
             self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)