Merge master

docs/README.md

@@ -0,0 +1,22 @@
+# Mitmproxy Documentation
+
+This directory houses the mitmproxy documentation available at <https://docs.mitmproxy.org/>.
+
+## Quick Start
+
+ 1. Install [hugo](https://gohugo.io/).
+ 2. Windows users: Depending on your git settings, you may need to manually create a symlink from 
+ /docs/src/examples to /examples.
+
+
+Now you can run `hugo server -D` in ./src.
+
+
+## Extended Install
+
+This is required to modify CSS files.
+
+ 1. Install node, yarn, and [modd](https://github.com/cortesi/modd).
+ 2. Run `yarn` in this directory to get node-sass.
+
+You can now run `modd` in this directory instead of running hugo directly.

docs/build

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-cd src; hugo

docs/build-archive

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+
+cd src
+DOCS_ARCHIVE=true hugo

docs/build-current

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+
+cd src
+hugo

docs/ci

@@ -1,13 +1,14 @@
 #!/bin/bash
+set -e
 
 # This script gets run from CI to render and upload docs
 
-./build
+./build-current
 
 # Only upload if we have defined credentials - we only have these defined for
 # trusted commits (i.e. not PRs).
 if [[ ! -z "${AWS_ACCESS_KEY_ID}" && $TRAVIS_BRANCH == "master" ]]; then
     aws s3 sync --acl public-read ./public s3://docs.mitmproxy.org/master
     aws cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
-        --paths "/master"
+        --paths "/master/*"
 fi

docs/setup

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -e
 
 aws configure set preview.cloudfront true
 aws --profile mitmproxy \

docs/src/config.toml

@@ -4,6 +4,7 @@ title = "mitmproxy.org docs"
 theme = "mitmproxydocs"
 publishDir = "../public"
 RelativeURLs = true
+googleAnalytics = "UA-4150636"
 
 [indexes]
   tag = "tags"

docs/src/content/_index.md

@@ -1,5 +1,6 @@
 ---
 title: "Introduction"
+layout: single
 menu:
     overview:
         weight: 1

docs/src/content/concepts-certificates.md

@@ -19,7 +19,7 @@ configure your target device with the correct proxy settings. Now start a
 browser on the device, and visit the magic domain **mitm.it**. You should see
 something like this:
 
-{{< figure src="/certinstall-webapp.png" >}}
+{{< figure src="/certinstall-webapp.png" class="has-border" >}}
 
 Click on the relevant icon, follow the setup instructions for the platform
 you're on and you are good to go.
@@ -32,8 +32,8 @@ reason. Below is a list of pointers to manual certificate installation
 documentation for some common platforms. The mitmproxy CA cert is located in
 `~/.mitmproxy` after it has been generated at the first start of mitmproxy.
 
-- [IOS](http://jasdev.me/intercepting-ios-traffic) On
-  iOS 10.3 and onwards, you also need to enable full trust for the mitmproxy
+- [IOS](http://jasdev.me/intercepting-ios-traffic)  
+  On iOS 10.3 and onwards, you also need to enable full trust for the mitmproxy
   root certificate:
     1. Go to Settings > General > About > Certificate Trust Settings.
     2. Under "Enable full trust for root certificates", turn on trust for
@@ -48,7 +48,7 @@ documentation for some common platforms. The mitmproxy CA cert is located in
 certutil.exe -importpfx Root mitmproxy-ca-cert.p12
 {{< / highlight >}}
   
-- [Mac OS X](https://support.apple.com/kb/PH7297?locale=en_US)
+- [Mac OS X](https://support.apple.com/kb/PH20129)
 - [Ubuntu/Debian]( https://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861)
 - [Mozilla Firefox](https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox)
 - [Chrome on Linux](https://stackoverflow.com/a/15076602/198996)
@@ -90,7 +90,7 @@ The files created by mitmproxy in the .mitmproxy directory are as follows:
 | mitmproxy-ca-cert.p12 | The certificate in PKCS12 format. For use on Windows.                                |
 | mitmproxy-ca-cert.cer | Same file as .pem, but with an extension expected by some Android devices.           |
 
-## Using a custom certificate
+## Using a custom server certificate
 
 You can use your own (leaf) certificate by passing the `--cert
 [domain=]path_to_certificate` option to mitmproxy. Mitmproxy then uses the
@@ -156,7 +156,7 @@ hostname, while using a filename allows a single specific certificate to be used
 for all SSL connections. Certificate files must be in the PEM format and should
 contain both the unencrypted private key and the certificate.
 
-### Multiple certs by Hostname
+### Multiple client certificates
 
 You can specify a directory to `--client-certs`, in which case the matching
 certificate is looked up by filename. So, if you visit example.org, mitmproxy

docs/src/content/howto-transparent.md

@@ -27,87 +27,50 @@ At the moment, mitmproxy supports transparent proxying on OSX Lion and above,
 and all current flavors of Linux.
 
 
-## Linux fully transparent mode
-
-By default mitmproxy will use its own local IP address for its server-side
-connections. In case this isn't desired, the --spoof-source-address argument can
-be used to use the client's IP address for server-side connections. The
-following config is required for this mode to work:
-
-{{< highlight bash  >}}
-CLIENT_NET=192.168.1.0/24
-TABLE_ID=100
-MARK=1
-
-echo "$TABLE_ID     mitmproxy" >> /etc/iproute2/rt_tables
-iptables -t mangle -A PREROUTING -d $CLIENT_NET -j MARK --set-mark $MARK
-iptables -t nat \
-    -A PREROUTING -p tcp -s $CLIENT_NET \
-    --match multiport --dports 80,443 -j \
-    REDIRECT --to-port 8080
-
-ip rule add fwmark $MARK lookup $TABLE_ID
-ip route add local $CLIENT_NET dev lo table $TABLE_ID
-{{< / highlight >}}
-
-This mode does require root privileges though. There's a wrapper in the examples
-directory called 'mitmproxy_shim.c', which will enable you to use this mode with
-dropped privileges. It can be used as follows:
-
-{{< highlight bash  >}}
-gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap
-sudo chown root:root mitmproxy_shim
-sudo chmod u+s mitmproxy_shim
-./mitmproxy_shim $(which mitmproxy) --mode transparent --set spoof-source-address
-{{< / highlight >}}
-
-
-
 ## Linux
 
 On Linux, mitmproxy integrates with the iptables redirection mechanism to
 achieve transparent mode.
 
-### 1. [Install the mitmproxy certificate on the test device]({{< relref "concepts-certificates" >}})
-
-### 2. Enable IP forwarding:
+### 1. Enable IP forwarding.
 
 {{< highlight bash  >}}
 sysctl -w net.ipv4.ip_forward=1
 sysctl -w net.ipv6.conf.all.forwarding=1
 {{< / highlight >}}
 
-You may also want to consider enabling this permanently in `/etc/sysctl.conf` or
-newly created `/etc/sysctl.d/mitmproxy.conf`, see
-[here](https://superuser.com/a/625852).
+This makes sure that your machine forwards packets instead of rejecting them.
 
-### 3. If your target machine is on the same physical network and you configured it to use a custom  gateway, disable ICMP redirects:
+If you want to persist this across reboots, you need to adjust your `/etc/sysctl.conf` or
+a newly created `/etc/sysctl.d/mitmproxy.conf` (see [here](https://superuser.com/a/625852)).
+
+### 2. Disable ICMP redirects.
 
 {{< highlight bash  >}}
 sysctl -w net.ipv4.conf.all.send_redirects=0
 {{< / highlight >}}
 
-You may also want to consider enabling this permanently in `/etc/sysctl.conf` or
-a newly created `/etc/sysctl.d/mitmproxy.conf`, see
-[here](https://superuser.com/a/625852).
+If your test device is on the same physical network, your machine shouldn't inform the device that 
+there's a shorter route available by skipping the proxy.
 
-### 4. Create an iptables ruleset that redirects the desired traffic to the mitmproxy port
+If you want to persist this across reboots, see above.
+
+### 3. Create an iptables ruleset that redirects the desired traffic to mitmproxy.
 
 Details will differ according to your setup, but the ruleset should look
 something like this:
 
 {{< highlight bash  >}}
-    iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080
-    iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080
-    ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080
-    ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080
+iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080
+iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080
+ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080
+ip6tables -t nat -A PREROUTING -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8080
 {{< / highlight >}}
 
-   You may also want to consider enabling this permanently with the
-`iptables-persistent` package, see
-[here](http://www.microhowto.info/howto/make_the_configuration_of_iptables_persistent_on_debian.html).
+If you want to persist this across reboots, you can use the `iptables-persistent` package (see
+[here](http://www.microhowto.info/howto/make_the_configuration_of_iptables_persistent_on_debian.html)).
 
-### 5. Fire up mitmproxy
+### 4. Fire up mitmproxy.
 
 You probably want a command like this:
 
@@ -118,24 +81,22 @@ mitmproxy --mode transparent --showhost
 The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells
  mitmproxy to use the value of the Host header for URL display.
 
-### 6. Finally, configure your test device
+### 5. Finally, configure your test device.
+
+Set the test device up to use the host on which mitmproxy is running as the default gateway and 
+[install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}).
 
-Set the test device up to use the host on which mitmproxy is running as the
-default gateway. For a detailed walkthrough, have a look at the [tutorial for
-transparently proxying VMs]({{< relref "howto-transparent-vms" >}}).
 
 
 ## OpenBSD
 
-### 1  [Install the mitmproxy certificate on the test device]({{< relref "concepts-certificates" >}})
-
-### 2. Enable IP forwarding
+### 1. Enable IP forwarding.
 
 {{< highlight bash  >}}
 sudo sysctl -w net.inet.ip.forwarding=1
 {{< / highlight >}}
 
-### 3. Place the following two lines in **/etc/pf.conf**
+### 2. Place the following two lines in **/etc/pf.conf**.
 
 {{< highlight none  >}}
 mitm_if = "re2"
@@ -146,19 +107,19 @@ These rules tell pf to divert all traffic from `$mitm_if` destined for port 80
 or 443 to the local mitmproxy instance running on port 8080. You should replace
 `$mitm_if` value with the interface on which your test device will appear.
 
-### 4. Enable the pf ruleset and enable it
+### 3. Configure pf with the rules.
 
 {{< highlight bash  >}}
 doas pfctl -f /etc/pf.conf
 {{< / highlight >}}
 
-And now enable it:
+### 4. And now enable it.
 
 {{< highlight bash  >}}
 doas pfctl -e
 {{< / highlight >}}
 
-### 5. Fire up mitmproxy
+### 5. Fire up mitmproxy.
 
 You probably want a command like this:
 
@@ -169,10 +130,11 @@ mitmproxy --mode transparent --showhost
 The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells
 mitmproxy to use the value of the Host header for URL display.
 
-### 6. Finally, configure your test device
+### 6. Finally, configure your test device.
+
+Set the test device up to use the host on which mitmproxy is running as the default gateway and 
+[install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}).
 
-Set the test device up to use the host on which mitmproxy is running as the
-default gateway.
 
 
 {{% note %}}
@@ -195,15 +157,13 @@ packet filter from the OpenBSD project, which mitmproxy uses to implement
 transparent mode on OSX. Note that this means we don't support transparent mode
 for earlier versions of OSX.
 
-### 1. [Install the mitmproxy certificate on the test device]({{< relref "concepts-certificates" >}})
-
-### 2. Enable IP forwarding
+### 1. Enable IP forwarding.
 
 {{< highlight bash  >}}
 sudo sysctl -w net.inet.ip.forwarding=1
 {{< / highlight >}}
 
-### 3. Place the following two lines in a file called, say, **pf.conf**
+### 2. Place the following two lines in a file called, say, **pf.conf**.
 
 
 {{< highlight none  >}}
@@ -214,19 +174,19 @@ These rules tell pf to redirect all traffic destined for port 80 or 443
 to the local mitmproxy instance running on port 8080. You should replace
 `en2` with the interface on which your test device will appear.
 
-### 4. Configure pf with the rules
+### 3. Configure pf with the rules.
 
 {{< highlight bash  >}}
 sudo pfctl -f pf.conf
 {{< / highlight >}}
 
-### 5. And now enable it
+### 4. And now enable it.
 
 {{< highlight bash  >}}
 sudo pfctl -e
 {{< / highlight >}}
 
-### 6. Configure sudoers to allow mitmproxy to access pfctl
+### 5. Configure sudoers to allow mitmproxy to access pfctl.
 
 Edit the file **/etc/sudoers** on your system as root. Add the following line to
 the end of the file:
@@ -240,7 +200,7 @@ state` as root without a password. This only allows inspection of the state
 table, so should not be an undue security risk. If you're special feel free to
 tighten the restriction up to the user running mitmproxy.
 
-### 7. Fire up mitmproxy
+### 6. Fire up mitmproxy.
 
 You probably want a command like this:
 
@@ -251,26 +211,25 @@ mitmproxy --mode transparent --showhost
 The `--mode transparent` flag turns on transparent mode, and the `--showhost` argument tells
 mitmproxy to use the value of the Host header for URL display.
 
-### 6. Finally, configure your test device
+### 7. Finally, configure your test device.
 
-Set the test device up to use the host on which mitmproxy is running as the
-default gateway.
+Set the test device up to use the host on which mitmproxy is running as the default gateway and 
+[install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}).
 
 {{% note %}}
 Note that the **rdr** rules in the pf.conf given above only apply to
 inbound traffic. **This means that they will NOT redirect traffic coming
 from the box running pf itself.** We can't distinguish between an
 outbound connection from a non-mitmproxy app, and an outbound connection
-from mitmproxy itself - if you want to intercept your OSX traffic, you
-should use an external host to run mitmproxy or see the work-around below. 
-PF is flexible to cater for a range of creative possibilities, like
+from mitmproxy itself. If you want to intercept your own macOS traffic, see the work-around below or use an external host to run mitmproxy. In fact, PF is
+flexible to cater for a range of creative possibilities, like
 intercepting traffic emanating from VMs. See the **pf.conf** man page
 for more.
 {{% /note %}}
 
 ### Work-around to redirect traffic originating from the machine itself
 
-Follow the steps **1, 2** as above. In step **3** change the file **pf.conf** to 
+Follow the steps **1, 2** as above. In step **3** change the contents of the file **pf.conf** to
 
 {{< highlight none >}}
 #The ports to redirect to proxy
@@ -303,3 +262,37 @@ Follow steps **4-6** above. This will redirect the packets from all users other
 {{< highlight bash  >}}
 sudo -u nobody mitmproxy --mode transparent --showhost
 {{< / highlight >}}
+
+## "Full" transparent mode on Linux
+
+By default mitmproxy will use its own local IP address for its server-side
+connections. In case this isn't desired, the --spoof-source-address argument can
+be used to use the client's IP address for server-side connections. The
+following config is required for this mode to work:
+
+{{< highlight bash  >}}
+CLIENT_NET=192.168.1.0/24
+TABLE_ID=100
+MARK=1
+
+echo "$TABLE_ID     mitmproxy" >> /etc/iproute2/rt_tables
+iptables -t mangle -A PREROUTING -d $CLIENT_NET -j MARK --set-mark $MARK
+iptables -t nat \
+    -A PREROUTING -p tcp -s $CLIENT_NET \
+    --match multiport --dports 80,443 -j \
+    REDIRECT --to-port 8080
+
+ip rule add fwmark $MARK lookup $TABLE_ID
+ip route add local $CLIENT_NET dev lo table $TABLE_ID
+{{< / highlight >}}
+
+This mode does require root privileges though. There's a wrapper in the examples
+directory called 'mitmproxy_shim.c', which will enable you to use this mode with
+dropped privileges. It can be used as follows:
+
+{{< highlight bash  >}}
+gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap
+sudo chown root:root mitmproxy_shim
+sudo chmod u+s mitmproxy_shim
+./mitmproxy_shim $(which mitmproxy) --mode transparent --set spoof-source-address
+{{< / highlight >}}

docs/src/layouts/_default/single.html

@@ -1,10 +1,12 @@
-{{ partial "header.html" . }}
-<div class="columns">
-    <div class="column is-one-quarter sidebody">
-        {{ partial "sidebar.html" . }}
+{{ partial "header" . }}
+<div class="columns container is-marginless">
+    <div id="sidebar" class="column is-one-quarter">
+        {{ partial "sidebar" . }}
     </div>
-    <div class="column content mainbody">
-        {{.Content}}
+    <div id="main" class="column content">
+        {{ partial "outdated" . }}
+        {{ partial "edit-on-github" . }}
+        {{ partial "add-anchors" .Content}}
     </div>
 </div>
 {{ partial "footer.html" . }}

docs/src/layouts/index.html

@@ -1,10 +0,0 @@
-{{ partial "header.html" . }}
-<div class="columns">
-    <div class="column is-one-quarter sidebody">
-        {{ partial "sidebar.html" . }}
-    </div>
-    <div class="column content mainbody">
-        {{.Content}}
-    </div>
-</div>
-{{ partial "footer.html" . }}

docs/src/layouts/partials/add-anchors.html

@@ -0,0 +1 @@
+{{ . | replaceRE "(<h[1-9] id=\"(.+?)\".*?>)(.+?</h[1-9]>)" "${1}<a class=\"anchor\" href=\"#${2}\">#&nbsp;&nbsp;</a>${3}" | safeHTML }}

docs/src/layouts/partials/edit-on-github.html

@@ -0,0 +1,9 @@
+{{ if and .IsPage (not (getenv "DOCS_ARCHIVE")) }}
+<a class="button is-small is-outlined is-link is-pulled-right"
+   target="_blank"
+   href="https://github.com/mitmproxy/mitmproxy/blob/master/docs/src/content/{{ .File.Path }}"
+>
+    Edit on GitHub
+</a>
+{{ end }}
+

docs/src/layouts/partials/outdated.html

@@ -0,0 +1,9 @@
+{{- if (getenv "DOCS_ARCHIVE") -}}
+<article class="message is-warning">
+    <div class="message-body">
+        You are not viewing the most up to date version of the documentation.
+        Click <a href="https://docs.mitmproxy.org/stable{{ .Page.URL }}">here</a>
+        to view the latest version.
+    </div>
+</article>
+{{- end -}}

docs/src/layouts/partials/sidebar.html

@@ -1,11 +1,10 @@
-<div class="sidebar">
-    <div class="brand">
-        <img src='{{"logo-docs.png" | relURL}}' alt="mitmproxy docs">
-    </div>
-    <div class="version">
-        <span class="tag is-info is-rounded is-medium">v3.x</span>
-    </div>
-    <aside class="menu">
+<div class="brand">
+    <a href="https://mitmproxy.org/">
+        <img src='{{"logo-docs.png" | relURL}}' alt="mitmproxy docs"/>
+    </a>
+
+</div>
+<nav class="menu">
     <p class="menu-label"> Overview </p>
     {{ partial "sidemenu" (dict "ctx" . "menuname" "overview") }}
 
@@ -20,5 +19,4 @@
 
     <p class="menu-label"> Tutorials </p>
     {{ partial "sidemenu" (dict "ctx" . "menuname" "tutes") }}
-    </aside>
-</div>
+</nav>

docs/src/themes/mitmproxydocs/layouts/partials/footer.html

@@ -1,2 +1,3 @@
+{{ template "_internal/google_analytics_async.html" . }}
 </body>
 </html>

docs/src/themes/mitmproxydocs/static/css/style.css

@@ -6717,9 +6717,17 @@ label.panel-block {
   background-color: whitesmoke;
   padding: 3rem 1.5rem 6rem; }
 
-.sidebody {
-  overflow-x: hidden;
-  overflow-y: scroll; }
+#sidebar {
+  background-color: #eee;
+  border-right: 1px solid #c1c1c1;
+  box-shadow: 0 0 20px rgba(50, 50, 50, 0.2) inset;
+  padding: 1.75rem; }
+  #sidebar .brand {
+    padding: 1rem 0;
+    text-align: center; }
+
+#main {
+  padding: 3rem; }
 
 .example {
   margin-bottom: 1em; }
@@ -6730,21 +6738,6 @@ label.panel-block {
     width: 100%;
     text-align: right; }
 
-.sidebar {
-  background-color: #F1F1F1; }
-  .sidebar .version {
-    padding: 1em; }
-  .sidebar .brand {
-    background-color: #303030;
-    color: #c0c0c0;
-    padding: 1em;
-    top: 0; }
-  .sidebar .menu {
-    padding: 1em; }
-
-.mainbody {
-  padding: 3em; }
-
 code {
   color: #1a9f1a;
   font-size: 0.875em;
@@ -6753,3 +6746,26 @@ code {
 .content h2 {
   padding-top: 1em;
   border-top: 1px solid #c0c0c0; }
+
+h1 .anchor, h2 .anchor, h3 .anchor, h4 .anchor, h5 .anchor, h6 .anchor {
+  display: inline-block;
+  width: 0;
+  margin-left: -1.5rem;
+  margin-right: 1.5rem;
+  transition: all 100ms ease-in-out;
+  opacity: 0; }
+
+h1:hover .anchor, h2:hover .anchor, h3:hover .anchor, h4:hover .anchor, h5:hover .anchor, h6:hover .anchor {
+  opacity: 1; }
+
+h1:target, h2:target, h3:target, h4:target, h5:target, h6:target {
+  color: #C93312; }
+  h1:target .anchor, h2:target .anchor, h3:target .anchor, h4:target .anchor, h5:target .anchor, h6:target .anchor {
+    opacity: 1;
+    color: #C93312; }
+
+.footnotes p {
+  display: inline; }
+
+figure.has-border img {
+  box-shadow: 0 0 20px 0 rgba(0, 0, 0, 0.25); }

docs/style/style.scss

@@ -10,9 +10,20 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox
 @import "../node_modules/bulma/sass/components/_all";
 @import "../node_modules/bulma/sass/layout/_all";
 
-.sidebody {
-    overflow-x: hidden;
-    overflow-y: scroll;
+#sidebar {
+    background-color: #eee;
+    border-right: 1px solid #c1c1c1;
+    box-shadow: 0 0 20px rgba(50, 50, 50, .2) inset;
+    padding: $column-gap + 1rem;
+
+    .brand {
+        padding: 1rem 0;
+        text-align: center;
+    }
+}
+
+#main {
+    padding: 3rem;
 }
 
 .example {
@@ -27,26 +38,6 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox
     margin-bottom: 1em;
 }
 
-.sidebar {
-    background-color: #F1F1F1;
-    .version {
-        padding: 1em;
-    }
-    .brand {
-        background-color: #303030;
-        color: #c0c0c0;
-        padding: 1em;
-        top: 0;
-    }
-    .menu {
-        padding: 1em;
-    }
-}
-
-.mainbody {
-  padding: 3em;
-}
-
 code {
     color: #1a9f1a;
     font-size: 0.875em;
@@ -59,3 +50,32 @@ code {
         border-top: 1px solid #c0c0c0;
     }
 }
+
+h1, h2, h3, h4, h5, h6 {
+    .anchor {
+        display: inline-block;
+        width: 0;
+        margin-left: -1.5rem;
+        margin-right: 1.5rem;
+        transition: all 100ms ease-in-out;
+        opacity: 0;
+    }
+    &:hover .anchor {
+        opacity: 1;
+    }
+    &:target {
+        color: $primary;
+        .anchor {
+            opacity: 1;
+            color: $primary
+        }
+    }
+}
+
+.footnotes p {
+    display: inline;
+}
+
+figure.has-border img {
+    box-shadow: 0 0 20px 0 rgba(0, 0, 0, 0.25);
+}

docs/upload-archive

@@ -1,4 +1,5 @@
-#!/bin/sh
+#!/bin/bash
+set -e
 
 if [[ $# -eq 0 ]] ; then
     echo "Please supply a version, e.g. 'v3'"
@@ -14,4 +15,4 @@ aws --profile mitmproxy \
     s3 sync --acl public-read ./public s3://docs.mitmproxy.org$SPATH
 aws --profile mitmproxy \
     cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
-    --paths "$SPATH"
+    --paths "$SPATH/*"

docs/upload-stable

@@ -1,8 +1,9 @@
-#!/bin/sh
+#!/bin/bash
+set -e
 
 aws configure set preview.cloudfront true
 aws --profile mitmproxy \
     s3 sync --acl public-read ./public s3://docs.mitmproxy.org/stable
 aws --profile mitmproxy \
     cloudfront create-invalidation --distribution-id E1TH3USJHFQZ5Q \
-    --paths "/stable"
+    --paths "/stable/*"

mitmproxy/addons/cut.py

@@ -129,7 +129,7 @@ class Cut:
             if isinstance(v, bytes):
                 fp.write(strutils.always_str(v))
             else:
-                fp.write("utf8")
+                fp.write(v)
             ctx.log.alert("Clipped single cut.")
         else:
             writer = csv.writer(fp)

mitmproxy/command.py

@@ -1,5 +1,5 @@
 """
-    This module manges and invokes typed commands.
+    This module manages and invokes typed commands.
 """
 import inspect
 import types
@@ -131,7 +131,12 @@ class CommandManager(mitmproxy.types._CommandBase):
         for i in dir(addon):
             if not i.startswith("__"):
                 o = getattr(addon, i)
-                if hasattr(o, "command_path"):
+                try:
+                    is_command = hasattr(o, "command_path")
+                except Exception:
+                    pass  # hasattr may raise if o implements __getattr__.
+                else:
+                    if is_command:
                         self.add(o.command_path, o)
 
     def add(self, path: str, func: typing.Callable):

mitmproxy/connections.py

@@ -1,18 +1,18 @@
-import time
-
 import os
+import time
 import typing
 import uuid
 
-from mitmproxy import stateobject, exceptions
 from mitmproxy import certs
+from mitmproxy import exceptions
+from mitmproxy import stateobject
 from mitmproxy.net import tcp
 from mitmproxy.net import tls
+from mitmproxy.utils import human
 from mitmproxy.utils import strutils
 
 
 class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
-
     """
     A client connection
 
@@ -72,11 +72,10 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         else:
             alpn = ""
 
-        return "<ClientConnection: {tls}{alpn}{host}:{port}>".format(
+        return "<ClientConnection: {tls}{alpn}{address}>".format(
             tls=tls,
             alpn=alpn,
-            host=self.address[0],
-            port=self.address[1],
+            address=human.format_address(self.address),
         )
 
     def __eq__(self, other):
@@ -161,7 +160,6 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
 
 
 class ServerConnection(tcp.TCPClient, stateobject.StateObject):
-
     """
     A server connection
 
@@ -209,11 +207,10 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
             )
         else:
             alpn = ""
-        return "<ServerConnection: {tls}{alpn}{host}:{port}>".format(
+        return "<ServerConnection: {tls}{alpn}{address}>".format(
             tls=tls,
             alpn=alpn,
-            host=self.address[0],
-            port=self.address[1],
+            address=human.format_address(self.address),
         )
 
     def __eq__(self, other):

mitmproxy/tools/console/statusbar.py

@@ -191,9 +191,7 @@ class StatusBar(urwid.WidgetWrap):
             r.append(("heading_key", "H"))
             r.append("eaders]")
         if len(self.master.options.replacements):
-            r.append("[")
-            r.append(("heading_key", "R"))
-            r.append("eplacing]")
+            r.append("[%d replacements]" % len(self.master.options.replacements))
         if creplay.count():
             r.append("[")
             r.append(("heading_key", "cplayback"))
@@ -228,10 +226,8 @@ class StatusBar(urwid.WidgetWrap):
             r.append("[")
             r.append(("heading_key", "u"))
             r.append(":%s]" % self.master.options.stickyauth)
-        if self.master.options.console_default_contentview != "auto":
-            r.append("[")
-            r.append(("heading_key", "M"))
-            r.append(":%s]" % self.master.options.console_default_contentview)
+        if self.master.options.console_default_contentview != 'auto':
+            r.append("[contentview:%s]" % (self.master.options.console_default_contentview))
         if self.master.options.has_changed("view_order"):
             r.append("[")
             r.append(("heading_key", "o"))

mitmproxy/utils/human.py

@@ -73,11 +73,13 @@ def format_timestamp_with_milli(s):
     return d.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
 
 
-def format_address(address: tuple) -> str:
+def format_address(address: typing.Optional[tuple]) -> str:
     """
     This function accepts IPv4/IPv6 tuples and
     returns the formatted address string with port number
     """
+    if address is None:
+        return "<no address>"
     try:
         host = ipaddress.ip_address(address[0])
         if host.is_unspecified:

setup.py

@@ -65,7 +65,7 @@ setup(
         "brotlipy>=0.7.0,<0.8",
         "certifi>=2015.11.20.1",  # no semver here - this should always be on the last release!
         "click>=6.2, <7",
-        "cryptography>=2.1.4,<2.2",
+        "cryptography>=2.1.4,<2.3",
         "h2>=3.0.1,<4",
         "hyperframe>=5.1.0,<6",
         "kaitaistruct>=0.7,<0.9",
@@ -77,7 +77,7 @@ setup(
         "pyperclip>=1.6.0, <1.7",
         "ruamel.yaml>=0.13.2, <0.16",
         "sortedcontainers>=1.5.4, <1.6",
-        "tornado>=4.3, <4.6",
+        "tornado>=4.3,<5.1",
         "urwid>=2.0.1,<2.1",
         "wsproto>=0.11.0,<0.12.0",
     ],
@@ -88,7 +88,7 @@ setup(
         'dev': [
             "flake8>=3.5, <3.6",
             "Flask>=0.10.1, <0.13",
-            "mypy>=0.570,<0.571",
+            "mypy>=0.580,<0.581",
             "pytest-cov>=2.5.1,<3",
             "pytest-faulthandler>=1.3.1,<2",
             "pytest-timeout>=1.2.1,<2",

test/bench/.gitignore

@@ -0,0 +1 @@
+results

test/bench/README.md

@@ -0,0 +1,56 @@
+
+This directory contains a set of tools for benchmarking and profiling mitmproxy.
+At the moment, this is simply to give developers a quick way to see the impact
+of their work. Eventually, this might grow into a performance dashboard with
+historical data, so we can track performance over time.
+
+
+# Setup
+
+Install the following tools:
+
+    go get -u github.com/rakyll/hey
+    go get github.com/cortesi/devd/cmd/devd
+
+You may also want to install snakeviz to make viewing profiles easier:
+
+    pip install snakeviz
+
+In one window, run the devd server:
+
+    ./backend
+
+
+# Running tests
+
+Each run consists of two files - a mitproxy invocation, and a traffic generator.
+Make sure the backend is started, then run the proxy:
+
+    ./simple.mitmproxy
+
+Now run the traffic generator:
+
+    ./simple.traffic
+
+After the run is done, quit the proxy with ctrl-c.
+
+
+# Reading results
+
+Results are placed in the ./results directory. You should see two files - a
+performance log from **hey**, and a profile. You can view the profile like so:
+
+    snakeviz ./results/simple.prof
+
+
+
+
+
+
+
+
+
+
+
+
+

test/bench/backend

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+devd -p 10001 .

test/bench/profiler.py

@@ -0,0 +1,25 @@
+import cProfile
+from mitmproxy import ctx
+
+
+class Profile:
+    """
+        A simple profiler addon.
+    """
+    def __init__(self):
+        self.pr = cProfile.Profile()
+
+    def load(self, loader):
+        loader.add_option(
+            "profile_path",
+            str,
+            "/tmp/profile",
+            "Destination for the run profile, saved at exit"
+        )
+        self.pr.enable()
+
+    def done(self):
+        self.pr.dump_stats(ctx.options.profile_path)
+
+
+addons = [Profile()]

test/bench/simple.mitmproxy

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+mkdir -p results
+mitmdump -p 10002 --mode reverse:http://devd.io:10001 \
+    -s ./profiler.py --set profile_path=./results/simple.prof

test/bench/simple.traffic

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+hey -disable-keepalive http://localhost:10002/profiler.py | tee ./results/simple.perf

test/mitmproxy/addons/test_onboarding.py

@@ -4,6 +4,10 @@ from mitmproxy.addons import onboarding
 from mitmproxy.test import taddons
 from .. import tservers
 
+import asyncio
+import tornado.platform.asyncio
+asyncio.set_event_loop_policy(tornado.platform.asyncio.AnyThreadEventLoopPolicy())
+
 
 class TestApp(tservers.HTTPProxyTest):
     def addons(self):

test/mitmproxy/test_command.py

@@ -309,6 +309,31 @@ class TDec:
         pass
 
 
+class TAttr:
+    def __getattr__(self, item):
+        raise IOError
+
+
+class TCmds(TAttr):
+    def __init__(self):
+        self.TAttr = TAttr()
+
+    @command.command("empty")
+    def empty(self) -> None:
+        pass
+
+
+def test_collect_commands():
+    """
+        This tests for the error thrown by hasattr()
+    """
+    with taddons.context() as tctx:
+        c = command.CommandManager(tctx.master)
+        a = TCmds()
+        c.collect_commands(a)
+        assert "empty" in c.commands
+
+
 def test_decorator():
     with taddons.context() as tctx:
         c = command.CommandManager(tctx.master)

test/mitmproxy/test_connections.py

@@ -38,6 +38,9 @@ class TestClientConnection:
         assert 'ALPN' not in repr(c)
         assert 'TLS' in repr(c)
 
+        c.address = None
+        assert repr(c)
+
     def test_tls_established_property(self):
         c = tflow.tclient_conn()
         c.tls_established = True
@@ -110,6 +113,9 @@ class TestServerConnection:
         c.tls_established = False
         assert 'TLS' not in repr(c)
 
+        c.address = None
+        assert repr(c)
+
     def test_tls_established_property(self):
         c = tflow.tserver_conn()
         c.tls_established = True

test/mitmproxy/utils/test_human.py

@@ -56,3 +56,4 @@ def test_format_address():
     assert human.format_address(("example.com", "54010")) == "example.com:54010"
     assert human.format_address(("::", "8080")) == "*:8080"
     assert human.format_address(("0.0.0.0", "8080")) == "*:8080"
+    assert human.format_address(None) == "<no address>"

web/README

@@ -1,6 +0,0 @@
-
-Starting up
-
-- npm install
-- gulp
-- run mitmweb and open http://localhost:8081/

web/README.md

@@ -0,0 +1,6 @@
+# Quick Start
+
+
+- Run `yarn` to install dependencies
+- Run `gulp` to start live-compilation.
+- Run `mitmweb` and open http://localhost:8081/