mirror of
https://github.com/Grasscutters/mitmproxy.git
synced 2025-01-30 23:09:44 +00:00
Merge remote-tracking branch 'base/master'
This commit is contained in:
Marcelo Glezer
commit
4952643a0d
@ -2,5 +2,5 @@
|
||||
branch = True
|
||||
|
||||
[report]
|
||||
omit = *contrib*, *tnetstring*, *platform*, *console*
|
||||
omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py
|
||||
include = *libmproxy*
|
||||
|
||||
50
CHANGELOG
50
CHANGELOG
@ -1,3 +1,53 @@
|
||||
|
||||
15 November 2014: mitmproxy 0.11.1:
|
||||
|
||||
* Bug fixes: connection leaks some crashes
|
||||
|
||||
|
||||
7 November 2014: mitmproxy 0.11:
|
||||
|
||||
* Performance improvements for mitmproxy console
|
||||
|
||||
* SOCKS5 proxy mode allows mitmproxy to act as a SOCKS5 proxy server
|
||||
|
||||
* Data streaming for response bodies exceeding a threshold
|
||||
(bradpeabody@gmail.com)
|
||||
|
||||
* Ignore hosts or IP addresses, forwarding both HTTP and HTTPS traffic
|
||||
untouched
|
||||
|
||||
* Finer-grained control of traffic replay, including options to ignore
|
||||
contents or parameters when matching flows (marcelo.glezer@gmail.com)
|
||||
|
||||
* Pass arguments to inline scripts
|
||||
|
||||
* Configurable size limit on HTTP request and response bodies
|
||||
|
||||
* Per-domain specification of interception certificates and keys (see
|
||||
--cert option)
|
||||
|
||||
* Certificate forwarding, relaying upstream SSL certificates verbatim (see
|
||||
--cert-forward)
|
||||
|
||||
* Search and highlighting for HTTP request and response bodies in
|
||||
mitmproxy console (pedro@worcel.com)
|
||||
|
||||
* Transparent proxy support on Windows
|
||||
|
||||
* Improved error messages and logging
|
||||
|
||||
* Support for FreeBSD in transparent mode, using pf (zbrdge@gmail.com)
|
||||
|
||||
* Content view mode for WBXML (davidshaw835@air-watch.com)
|
||||
|
||||
* Better documentation, with a new section on proxy modes
|
||||
|
||||
* Generic TCP proxy mode
|
||||
|
||||
* Countless bugfixes and other small improvements
|
||||
|
||||
|
||||
|
||||
28 January 2014: mitmproxy 0.10:
|
||||
|
||||
* Support for multiple scripts and multiple script arguments
|
||||
|
||||
68
CONTRIBUTORS
68
CONTRIBUTORS
@ -1,51 +1,65 @@
|
||||
854 Aldo Cortesi
|
||||
64 Maximilian Hils
|
||||
902 Aldo Cortesi
|
||||
323 Maximilian Hils
|
||||
18 Henrik Nordstrom
|
||||
13 Thomas Roth
|
||||
12 Pedro Worcel
|
||||
11 Stephen Altamirano
|
||||
10 András Veres-Szentkirályi
|
||||
8 Jason A. Novak
|
||||
8 Rouli
|
||||
8 Jason A. Novak
|
||||
7 Alexis Hildebrandt
|
||||
6 Pedro Worcel
|
||||
5 Tomaz Muraus
|
||||
5 Brad Peabody
|
||||
5 Matthias Urlichs
|
||||
4 root
|
||||
4 Bryan Bishop
|
||||
4 Marc Liyanage
|
||||
4 Valtteri Virtanen
|
||||
3 Kyle Manna
|
||||
4 Bryan Bishop
|
||||
3 Chris Neasbitt
|
||||
2 alts
|
||||
2 Heikki Hannikainen
|
||||
2 Jim Lloyd
|
||||
3 Zack B
|
||||
3 Eli Shvartsman
|
||||
3 Kyle Manna
|
||||
2 Michael Frister
|
||||
2 Bennett Blodinger
|
||||
2 Jim Lloyd
|
||||
2 Rob Wills
|
||||
2 Jaime Soriano Pastor
|
||||
2 israel
|
||||
2 Jaime Soriano Pastor
|
||||
2 Heikki Hannikainen
|
||||
2 Mark E. Haase
|
||||
2 alts
|
||||
1 davidpshaw
|
||||
1 deployable
|
||||
1 joebowbeer
|
||||
1 meeee
|
||||
1 phil plante
|
||||
1 Michael Bisbjerg
|
||||
1 Andy Smith
|
||||
1 Dan Wilbraham
|
||||
1 David Shaw
|
||||
1 Eric Entzel
|
||||
1 Felix Wolfsteller
|
||||
1 Henrik Nordström
|
||||
1 Ivaylo Popov
|
||||
1 JC
|
||||
1 Jakub Nawalaniec
|
||||
1 James Billingham
|
||||
1 Jean Regisser
|
||||
1 Kit Randel
|
||||
1 Marcelo Glezer
|
||||
1 Mathieu Mitchell
|
||||
1 Mikhail Korobov
|
||||
1 Nicolas Esteves
|
||||
1 Oleksandr Sheremet
|
||||
1 Paul
|
||||
1 Rich Somerfield
|
||||
1 Rory McCann
|
||||
1 Felix Wolfsteller
|
||||
1 Rune Halvorsen
|
||||
1 Sahn Lam
|
||||
1 Eric Entzel
|
||||
1 Dan Wilbraham
|
||||
1 Seppo Yli-Olli
|
||||
1 Sergey Chipiga
|
||||
1 Steven Van Acker
|
||||
1 Ulrich Petri
|
||||
1 Andy Smith
|
||||
1 Vyacheslav Bakhmutov
|
||||
1 Yuangxuan Wang
|
||||
1 capt8bit
|
||||
1 joebowbeer
|
||||
1 meeee
|
||||
1 James Billingham
|
||||
1 Jakub Nawalaniec
|
||||
1 JC
|
||||
1 Kit Randel
|
||||
1 phil plante
|
||||
1 Mathieu Mitchell
|
||||
1 Ivaylo Popov
|
||||
1 Henrik Nordström
|
||||
1 Michael Bisbjerg
|
||||
1 Nicolas Esteves
|
||||
1 Oleksandr Sheremet
|
||||
|
||||
@ -2,10 +2,7 @@ include mitmproxy mitmdump
|
||||
include LICENSE CHANGELOG CONTRIBUTORS README.txt
|
||||
exclude README.mkd
|
||||
recursive-include examples *
|
||||
recursive-exclude examples *.pyc *.pyo *.swo *.swp
|
||||
recursive-include doc *
|
||||
recursive-exclude doc *.pyc *.pyo *.swo *.swp
|
||||
recursive-include test *
|
||||
recursive-exclude test *.pyc *.pyo *.swo *.swp
|
||||
recursive-include libmproxy *
|
||||
recursive-exclude libmproxy *.pyc *.pyo *.swo *.swp
|
||||
recursive-exclude * *.pyc *.pyo *.swo *.swp
|
||||
46
README.mkd
46
README.mkd
@ -13,6 +13,9 @@ mitmproxy.org website:
|
||||
[mitmproxy.org](http://mitmproxy.org).
|
||||
|
||||
|
||||
You can find complete directions for installing mitmproxy [here](http://mitmproxy.org/doc/install.html).
|
||||
|
||||
|
||||
Features
|
||||
--------
|
||||
|
||||
@ -26,17 +29,17 @@ Features
|
||||
- SSL certificates for interception are generated on the fly.
|
||||
- And much, much more.
|
||||
|
||||
|
||||
Installation
|
||||
------------
|
||||
__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. On Windows,
|
||||
only mitmdump is supported, which does not have a graphical user interface.
|
||||
|
||||
|
||||
The recommended way to install mitmproxy is running <code>pip install mitmproxy</code>.
|
||||
For convenience, we provide binary packages on [mitmproxy.org](http://mitmproxy.org/).
|
||||
|
||||
Hacking
|
||||
-------
|
||||
|
||||
|
||||
Requirements
|
||||
------------
|
||||
### Requirements
|
||||
|
||||
|
||||
* [Python](http://www.python.org) 2.7.x.
|
||||
* [netlib](http://pypi.python.org/pypi/netlib), version matching mitmproxy.
|
||||
@ -49,28 +52,35 @@ Optional packages for extended content decoding:
|
||||
* [cssutils](http://cthedot.de/cssutils/) version 1.0 or newer.
|
||||
|
||||
For convenience, all optional dependencies can be installed with
|
||||
`pip install mitmproxy[contenviews]`
|
||||
|
||||
__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. On Windows,
|
||||
only mitmdump is supported, which does not have a graphical user interface.
|
||||
`pip install "mitmproxy[contentviews]"`
|
||||
|
||||
### Setting up a dev environment
|
||||
|
||||
Hacking
|
||||
-------
|
||||
The following procedure is recommended to set up your dev environment:
|
||||
|
||||
The following components are needed if you plan to hack on mitmproxy:
|
||||
|
||||
* The test suite requires the `dev` extra requirements listed in [setup.py](https://github.com/mitmproxy/mitmproxy/blob/master/setup.py) and [pathod](http://pathod.net), version matching mitmproxy.
|
||||
* Rendering the documentation requires [countershape](http://github.com/cortesi/countershape).
|
||||
|
||||
For convenience, the following procedure is recommended to set up your environment:
|
||||
```
|
||||
$ git clone https://github.com/mitmproxy/mitmproxy.git
|
||||
$ cd mitmproxy
|
||||
$ pip install --src . -r requirements.txt
|
||||
```
|
||||
|
||||
This installs the latest GitHub versions of mitmproxy, netlib and pathod into `mitmproxy/`. All other development dependencies save countershape are installed into their usual locations.
|
||||
|
||||
|
||||
### Testing
|
||||
|
||||
The test suite requires the `dev` extra requirements listed in [setup.py](https://github.com/mitmproxy/mitmproxy/blob/master/setup.py) and [pathod](http://pathod.net), version matching mitmproxy. Install these with:
|
||||
|
||||
`pip install "mitmproxy[dev]"`
|
||||
|
||||
|
||||
Please ensure that all patches are accompanied by matching changes in the test
|
||||
suite. The project maintains 100% test coverage.
|
||||
|
||||
|
||||
### Docs
|
||||
|
||||
Rendering the documentation requires [countershape](http://github.com/cortesi/countershape). After installation, you can render the documentation to the doc like this:
|
||||
|
||||
`cshape doc-src doc`
|
||||
|
||||
@ -17,12 +17,14 @@
|
||||
$!nav("serverreplay.html", this, state)!$
|
||||
$!nav("setheaders.html", this, state)!$
|
||||
$!nav("passthrough.html", this, state)!$
|
||||
$!nav("sticky.html", this, state)!$
|
||||
$!nav("proxyauth.html", this, state)!$
|
||||
$!nav("reverseproxy.html", this, state)!$
|
||||
$!nav("responsestreaming.html", this, state)!$
|
||||
$!nav("socksproxy.html", this, state)!$
|
||||
$!nav("sticky.html", this, state)!$
|
||||
$!nav("tcpproxy.html", this, state)!$
|
||||
$!nav("upstreamproxy.html", this, state)!$
|
||||
$!nav("upstreamcerts.html", this, state)!$
|
||||
$!nav("proxyauth.html", this, state)!$
|
||||
$!nav("responsestreaming.html", this, state)!$
|
||||
|
||||
|
||||
<li class="nav-header">Installing Certificates</li>
|
||||
|
||||
@ -9,9 +9,11 @@ pages = [
|
||||
Page("replacements.html", "Replacements"),
|
||||
Page("responsestreaming.html", "Response Streaming"),
|
||||
Page("reverseproxy.html", "Reverse proxy mode"),
|
||||
Page("socksproxy.html", "SOCKS Mode"),
|
||||
Page("setheaders.html", "Set Headers"),
|
||||
Page("serverreplay.html", "Server-side replay"),
|
||||
Page("sticky.html", "Sticky cookies and auth"),
|
||||
Page("tcpproxy.html", "TCP Proxy"),
|
||||
Page("upstreamcerts.html", "Upstream Certs"),
|
||||
Page("upstreamproxy.html", "Upstream proxy mode"),
|
||||
]
|
||||
@ -1,13 +1,12 @@
|
||||
There are a couple of reasons why you may want to exempt some traffic from mitmproxy's interception mechanism:
|
||||
There are two main reasons why you may want to exempt some traffic from mitmproxy's interception mechanism:
|
||||
|
||||
- **Certificate pinning:** Some traffic is is protected using
|
||||
[certificate pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's
|
||||
interception leads to errors. For example, Windows Update or the Apple App Store fail to work if mitmproxy is active.
|
||||
- **Non-HTTP traffic:** WebSockets or other non-http protocols are not supported by mitmproxy yet. You can exempt the
|
||||
domain from processing, which would otherwise fail.
|
||||
- **Convenience:** You really don't care about some parts of the traffic and just want them to go away.
|
||||
|
||||
If you want to ignore traffic from mitmproxy's processing because of large response bodies, check out the
|
||||
If you want to peek into (SSL-protected) non-HTTP connections, check out the [tcp proxy](@!urlTo("tcpproxy.html")!@) feature.
|
||||
If you want to ignore traffic from mitmproxy's processing because of large response bodies, take a look at the
|
||||
[response streaming](@!urlTo("responsestreaming.html")!@) feature.
|
||||
|
||||
## How it works
|
||||
@ -74,4 +73,9 @@ Here are some other examples for ignore patterns:
|
||||
--ignore 17\.178\.\d+\.\d+:443
|
||||
</pre>
|
||||
|
||||
### See Also
|
||||
|
||||
- [TCP Proxy](@!urlTo("tcpproxy.html")!@)
|
||||
- [Response Streaming](@!urlTo("responsestreaming.html")!@)
|
||||
|
||||
[^explicithttp]: This stems from an limitation of explicit HTTP proxying: A single connection can be re-used for multiple target domains - a <code>GET http://example.com/</code> request may be followed by a <code>GET http://evil.com/</code> request on the same connection. If we start to ignore the connection after the first request, we would miss the relevant second one.
|
||||
@ -47,4 +47,8 @@ When response streaming is enabled, portions of the code which would have otherw
|
||||
on the response body will see an empty response body instead (<code>libmproxy.protocol.http.CONTENT_MISSING</code>). Any modifications will be ignored.
|
||||
|
||||
Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a <code>Transfer-Encoding:
|
||||
chunked</code> header, the response will be streamed one chunk at a time.
|
||||
chunked</code> header, the response will be streamed one chunk at a time.
|
||||
|
||||
### See Also
|
||||
|
||||
- [Ignore Domains](@!urlTo("passthrough.html")!@)
|
||||
|
||||
@ -7,10 +7,46 @@ mitmproxy forwards HTTP proxy requests to an upstream proxy server.
|
||||
<table class="table">
|
||||
<tbody>
|
||||
<tr>
|
||||
<th width="20%">command-line</th> <td>-R http[s]://hostname[:port]</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>mitmproxy shortcut</th> <td><b>P</b></td>
|
||||
<th width="20%">command-line</th> <td>-R <i>schema</i>://hostname[:port]</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
Here, **schema** is one of http, https, http2https or https2http. The latter
|
||||
two extended schema specifications control the use of HTTP and HTTPS on
|
||||
mitmproxy and the upstream server. You can indicate that mitmproxy should use
|
||||
HTTP, and the upstream server uses HTTPS like this:
|
||||
|
||||
http2https://hostname:port
|
||||
|
||||
And you can indicate that mitmproxy should use HTTPS while the upstream
|
||||
service uses HTTP like this:
|
||||
|
||||
https2http://hostname:port
|
||||
|
||||
|
||||
### Host Header
|
||||
|
||||
In reverse proxy mode, mitmproxy does not rewrite the host header. While often useful, this
|
||||
may lead to issues with public web servers. For example, consider the following scenario:
|
||||
|
||||
$ python mitmdump -d -R http://example.com/ &
|
||||
$ curl http://localhost:8080/
|
||||
|
||||
>> GET https://example.com/
|
||||
Host: localhost:8080
|
||||
User-Agent: curl/7.35.0
|
||||
[...]
|
||||
|
||||
<< 404 Not Found 345B
|
||||
|
||||
Since the Host header doesn't match <samp>example.com</samp>, an error is returned.<br>
|
||||
There are two ways to solve this:
|
||||
<ol>
|
||||
<li>Modify the hosts file of your OS so that example.com resolves to 127.0.0.1.</li>
|
||||
<li>
|
||||
Instruct mitmproxy to rewrite the host header by passing <kbd>‑‑setheader :~q:Host:example.com</kbd>.
|
||||
However, keep in mind that absolute URLs within the returned document or HTTP redirects will cause the client application
|
||||
to bypass the proxy.
|
||||
</li>
|
||||
</ol>
|
||||
10
doc-src/features/socksproxy.html
Normal file
10
doc-src/features/socksproxy.html
Normal file
@ -0,0 +1,10 @@
|
||||
|
||||
In this mode, mitmproxy acts as a SOCKS5 proxy server.
|
||||
|
||||
<table class="table">
|
||||
<tbody>
|
||||
<tr>
|
||||
<th width="20%">command-line</th> <td>--socks</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
30
doc-src/features/tcpproxy.html
Normal file
30
doc-src/features/tcpproxy.html
Normal file
@ -0,0 +1,30 @@
|
||||
WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt hostnames from
|
||||
processing, so that mitmproxy acts as a generic TCP forwarder. This feature is closely related to the
|
||||
[ignore domains](@!urlTo("passthrough.html")!@) functionality, but differs in two important aspects:
|
||||
|
||||
- The raw TCP messages are printed to the event log.
|
||||
- SSL connections will be intercepted.
|
||||
|
||||
Please note that message interception or modification are not possible yet.
|
||||
If you are not interested in the raw TCP messages, you should use the ignore domains feature.
|
||||
|
||||
## How it works
|
||||
|
||||
|
||||
<table class="table">
|
||||
<tbody>
|
||||
<tr>
|
||||
<th width="20%">command-line</th> <td>--tcp HOST</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>mitmproxy shortcut</th> <td><b>T</b></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
For a detailed description on the structure of the hostname pattern, please refer to the [Ignore Domains](@!urlTo("passthrough.html")!@) feature.
|
||||
|
||||
### See Also
|
||||
|
||||
- [Ignore Domains](@!urlTo("passthrough.html")!@)
|
||||
- [Response Streaming](@!urlTo("responsestreaming.html")!@)
|
||||
@ -9,8 +9,19 @@ mitmproxy forwards ordinary HTTP requests to an upstream server.
|
||||
<tr>
|
||||
<th width="20%">command-line</th> <td>-U http://hostname[:port]</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>mitmproxy shortcut</th> <td><b>U</b></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
Here, **schema** is one of http, https, http2https or https2http. The latter
|
||||
two extended schema specifications control the use of HTTP and HTTPS on
|
||||
mitmproxy and the upstream server. You can indicate that mitmproxy should use
|
||||
HTTP, and the upstream server uses HTTPS like this:
|
||||
|
||||
http2https://hostname:port
|
||||
|
||||
And you can indicate that mitmproxy should use HTTPS while the upstream
|
||||
service uses HTTP like this:
|
||||
|
||||
https2http://hostname:port
|
||||
|
||||
|
||||
|
||||
@ -1,4 +1,27 @@
|
||||
|
||||
@!index_contents!@
|
||||
__mitmproxy__ is an interactive, SSL-capable man-in-the-middle proxy for HTTP
|
||||
with a console interface.
|
||||
|
||||
__mitmdump__ is the command-line version of mitmproxy. Think tcpdump for HTTP.
|
||||
|
||||
__libmproxy__ is the library that mitmproxy and mitmdump are built on.
|
||||
|
||||
Documentation, tutorials and distribution packages can be found on the
|
||||
mitmproxy.org website:
|
||||
|
||||
[mitmproxy.org](http://mitmproxy.org).
|
||||
|
||||
|
||||
Features
|
||||
--------
|
||||
|
||||
- Intercept HTTP requests and responses and modify them on the fly.
|
||||
- Save complete HTTP conversations for later replay and analysis.
|
||||
- Replay the client-side of an HTTP conversations.
|
||||
- Replay HTTP responses of a previously recorded server.
|
||||
- Reverse proxy mode to forward traffic to a specified server.
|
||||
- Transparent proxy mode on OSX and Linux.
|
||||
- Make scripted changes to HTTP traffic using Python.
|
||||
- SSL certificates for interception are generated on the fly.
|
||||
- And much, much more.
|
||||
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
import os, sys, datetime
|
||||
import os
|
||||
import sys
|
||||
import datetime
|
||||
import countershape
|
||||
from countershape import Page, Directory, PythonModule, markup, model
|
||||
from countershape import Page, Directory, markup, model
|
||||
import countershape.template
|
||||
sys.path.insert(0, "..")
|
||||
from libmproxy import filt, version
|
||||
@ -23,18 +25,18 @@ ns.docMaintainer = "Aldo Cortesi"
|
||||
ns.docMaintainerEmail = "aldo@corte.si"
|
||||
ns.copyright = u"\u00a9 mitmproxy project, %s" % datetime.date.today().year
|
||||
|
||||
|
||||
def mpath(p):
|
||||
p = os.path.join(MITMPROXY_SRC, p)
|
||||
return os.path.expanduser(p)
|
||||
|
||||
with open(mpath("README.mkd")) as f:
|
||||
readme = f.read()
|
||||
ns.index_contents = readme.split("\n", 1)[1] #remove first line (contains build status)
|
||||
|
||||
def example(s):
|
||||
d = file(mpath(s)).read().rstrip()
|
||||
extemp = """<div class="example">%s<div class="example_legend">(%s)</div></div>"""
|
||||
return extemp%(countershape.template.Syntax("py")(d), s)
|
||||
|
||||
|
||||
ns.example = example
|
||||
|
||||
|
||||
@ -73,6 +75,7 @@ def nav(page, current, state):
|
||||
ns.nav = nav
|
||||
ns.navbar = countershape.template.File(None, "_nav.html")
|
||||
|
||||
|
||||
pages = [
|
||||
Page("index.html", "Introduction"),
|
||||
Page("install.html", "Installation"),
|
||||
|
||||
@ -1,40 +1,33 @@
|
||||
|
||||
## Installing from source
|
||||
|
||||
The preferred way to install mitmproxy - whether you're installing the latest
|
||||
release or from source - is to use [pip](http://www.pip-installer.org/). If you
|
||||
don't already have pip on your system, you can find installation instructions
|
||||
[here](http://www.pip-installer.org/en/latest/installing.html).
|
||||
|
||||
|
||||
## Installing the latest release
|
||||
|
||||
A single command will download and install the latest release of mitmproxy,
|
||||
along with all its dependencies:
|
||||
|
||||
<pre class="terminal">
|
||||
pip install mitmproxy
|
||||
</pre>
|
||||
|
||||
If you also want to install the optional packages AMF, protobuf and CSS
|
||||
content views, do this:
|
||||
|
||||
## Installing from source
|
||||
|
||||
When installing from source, the easiest method is still to use pip. In this
|
||||
case run:
|
||||
|
||||
<pre class="terminal">
|
||||
pip install /path/to/source
|
||||
pip install "mitmproxy[contentviews]"
|
||||
</pre>
|
||||
|
||||
Note that if you're installing current git master, you will also have to
|
||||
install the current git master of [netlib](http://github.com/mitmproxy/netlib) by
|
||||
hand.
|
||||
|
||||
## OSX
|
||||
|
||||
The easiest way to get up and running on OSX is to download the pre-built
|
||||
binary packages from [mitmproxy.org](http://mitmproxy.org). If you still want
|
||||
to install using pip, there are a few things to keep in mind:
|
||||
|
||||
- If you're running a Python interpreter installed with homebrew (or similar),
|
||||
you may have to install some dependencies by hand.
|
||||
- Make sure that XCode is installed from the App Store, and that the
|
||||
command-line tools have been downloaded (XCode/Preferences/Downloads).
|
||||
- Now use __pip__ to do the installation, as above.
|
||||
|
||||
There are a few bits of customization you might want to do to make mitmproxy
|
||||
comfortable to use on OSX. The default color scheme is optimized for a dark
|
||||
@ -64,8 +57,3 @@ from source:
|
||||
- libxslt1-dev
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,210 +1,222 @@
|
||||
Mitmproxy comes with several modes of operation, which allow you to use mitmproxy in a variety of scenarios.
|
||||
This documents briefly explains each mode and possible setups.
|
||||
<hr>
|
||||
Mitmproxy has four modes of operation:
|
||||
<ul>
|
||||
<li>Regular Mode (this is what you get by default)</li>
|
||||
<li>Transparent Mode</li>
|
||||
<li>Reverse Proxy Mode</li>
|
||||
<li>Upstream Proxy Mode</li>
|
||||
</ul>
|
||||
|
||||
<p>Now, which one should you pick? Use this flow chart:
|
||||
</p>
|
||||
Mitmproxy has four modes of operation that allow you to use mitmproxy in a
|
||||
variety of scenarios:
|
||||
|
||||
<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"><br><br>
|
||||
- **Regular** (the default)
|
||||
- **Transparent**
|
||||
- **Reverse Proxy**
|
||||
- **Upstream Proxy**
|
||||
|
||||
Now, which one should you pick? Use this flow chart:
|
||||
|
||||
<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
|
||||
|
||||
<div class="page-header">
|
||||
<h1>Regular Proxy</h1>
|
||||
</div>
|
||||
|
||||
Mitmproxy's regular mode it the most simple one and the easiest to set up.
|
||||
Mitmproxy's regular mode is the simplest and the easiest to set up.
|
||||
|
||||
<ol>
|
||||
<li>Start mitmproxy.</li>
|
||||
<li>Configure your client to use mitmproxy. This means that you either adjust the proxy setting of your local browser
|
||||
or point an external device to your proxy (which should look like
|
||||
<a href="@!urlTo('screenshots/ios-manual.png')!@">this</a>).</li>
|
||||
<li>Quick Check: You can already visit an unencrypted HTTP site over the proxy.</li>
|
||||
<li>Open the magic domain <strong>mitm.it</strong> and install the certificate for your device.</li>
|
||||
</ol>
|
||||
1. Start mitmproxy.
|
||||
2. Configure your client to use mitmproxy. For instance on IOS, the settings might look like <a href="@!urlTo('screenshots/ios-manual.png')!@">this</a>.
|
||||
3. Quick Check: You should already be able to visit an unencrypted HTTP site
|
||||
through the proxy.
|
||||
4. Open the magic domain <strong>mitm.it</strong> and install the certificate for your device.
|
||||
|
||||
<div class="well">
|
||||
<strong>Heads Up:</strong> Unfortunately, some applications prefer to bypass the HTTP proxy settings of the system -
|
||||
Android applications are a common example. In these cases, you need to use mitmproxy's transparent mode.
|
||||
<strong>Heads Up:</strong> Unfortunately, some applications bypass the
|
||||
system HTTP proxy settings - Android applications are a common example. In
|
||||
these cases, you need to use mitmproxy's transparent mode.
|
||||
</div>
|
||||
|
||||
<p>If you are proxying an external device, your network will probably look like this:</p>
|
||||
If you are proxying an external device, your network will probably look like this:
|
||||
|
||||
<img src="@!urlTo('schematics/proxy-modes-regular.png')!@">
|
||||
<br><br>
|
||||
<p>The square brackets signify the source and destination IP addresses. Your client explicitly connects
|
||||
to mitmproxy and mitmproxy explicitly connects to the target server.
|
||||
</p>
|
||||
|
||||
The square brackets signify the source and destination IP addresses. Your
|
||||
client explicitly connects to mitmproxy and mitmproxy explicitly connects
|
||||
to the target server.
|
||||
|
||||
<div class="page-header">
|
||||
<h1>Transparent Proxy</h1>
|
||||
</div>
|
||||
|
||||
When a transparent proxy is used, traffic is redirected into a proxy at the network layer, without any client
|
||||
configuration being required. This makes transparent proxying ideal for those situations where you can't change client
|
||||
behaviour. The basic principle is that mitmproxy sits somewhere on the line from the client to the internet and
|
||||
transparently intercepts the request. In the graphic below, a machine running mitmproxy has been inserted between
|
||||
the router and the internet:
|
||||
In transparent mode, traffic is directed into a proxy at the network layer,
|
||||
without any client configuration required. This makes transparent proxying
|
||||
ideal for situations where you can't change client behaviour. In the graphic
|
||||
below, a machine running mitmproxy has been inserted between the router and
|
||||
the internet:
|
||||
|
||||
<a href="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
|
||||
<img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@"></a>
|
||||
<p>The square brackets signify the source and destination IP addresses. Round brackets mark the next
|
||||
hop on the <strong>Ethernet</strong>/data link layer. This distinction is important to make: When the packet arrives
|
||||
at the mitmproxy machine, it must still be addressed to the target server. In other words: A simple IP redirect on
|
||||
the router does not work - this would remove the target information, leaving mitmproxy unable to
|
||||
determine the real destination.
|
||||
</p>
|
||||
<img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
|
||||
</a>
|
||||
|
||||
The square brackets signify the source and destination IP addresses. Round
|
||||
brackets mark the next hop on the *Ethernet/data link* layer. This distinction
|
||||
is important: when the packet arrives at the mitmproxy machine, it must still
|
||||
be addressed to the target server. This means that Network Address Translation
|
||||
should not be applied before the traffic reaches mitmproxy, since this would
|
||||
remove the target information, leaving mitmproxy unable to determine the real
|
||||
destination.
|
||||
|
||||
<a href="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@">
|
||||
<img src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
|
||||
|
||||
<h2>Common Configurations</h2>
|
||||
|
||||
The first graphic is a little bit idealistic: Usually, you'll have your local wireless lan network and no
|
||||
machines between your router and the internet. Fortunately, there are other ways to configure your network:
|
||||
(a) Configuring the client to use a custom gateway/router/"next hop", (b) Implementing custom routing on the router
|
||||
or (c) setting up a separate wireless network router which gets proxied.
|
||||
There are of course other options, but we'll look at these three. In most cases, setting (a) is recommended due to its
|
||||
ease of use.
|
||||
There are many ways to configure your network for transparent proxying. We'll
|
||||
look at three common scenarios:
|
||||
|
||||
1. Configuring the client to use a custom gateway/router/"next hop"
|
||||
2. Implementing custom routing on the router
|
||||
|
||||
In most cases, the first option is recommended due to its ease of use.
|
||||
|
||||
<h3>(a) Custom Gateway</h3>
|
||||
|
||||
<p>Looking at your local home network, it's clear what happens if you enter "example.com" into your address bar: After you
|
||||
press enter, your OS sends a packet to your router, which then sends this to your ISP, which then sends it to some
|
||||
Tier-1 carrier, which then sends it... I think you get the idea. The important part for us is the first step here:
|
||||
Your machine is configured to use your router as the next hop. Your router certainly doesn't host example.com, but your
|
||||
machine knows that your router will forward it upstream. On the technical level, your router probably provides a DHCP
|
||||
server, which instructs all clients to use his address as the <em>Default Gateway</em> for connections that leave the
|
||||
current subnet (your local network).</p>
|
||||
<p>
|
||||
How does this help us? Here comes our trick: By configuring the client to use our machine as its Gateway, all traffic
|
||||
will be sent to our machine, which then forwards it to the router. This provides us with the scenario we'd like to have,
|
||||
namely packets on our doorstep that are addressed for someone else:
|
||||
</p>
|
||||
One simple way to get traffic to the mitmproxy machine with the destination IP
|
||||
intact, is to simply configure the client with the mitmproxy box as the
|
||||
default gateway.
|
||||
|
||||
<a href="@!urlTo('schematics/proxy-modes-transparent-2.png')!@">
|
||||
<img src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
|
||||
|
||||
Given this concept, we can set up mitmproxy:
|
||||
<ol>
|
||||
<li>Configure your proxy machine for transparent mode.<br>You can find instructions
|
||||
in the <em>Transparent Proxying</em> section of the mitmproxy docs.</li>
|
||||
<li>Configure your client to use your proxy machine's IP as the default gateway. This setting is usually called
|
||||
<em>Standard Gateway, Router</em> or something along these lines
|
||||
(<a href="@!urlTo('screenshots/ios-gateway.png')!@">iOS screenshot</a>).</li>
|
||||
<li>Quick Check: You can already visit an unencrypted HTTP site over the proxy.</li>
|
||||
<li>Open the magic domain <strong>mitm.it</strong> and install the certificate for your device.</li>
|
||||
</ol>
|
||||
In this scenario, we would:
|
||||
|
||||
- Configure the proxy machine for transparent mode. You can find instructions
|
||||
in the <em>Transparent Proxying</em> section of the mitmproxy docs.
|
||||
|
||||
- Configure the client to use the proxy machine's IP as the default gateway.
|
||||
<a href="@!urlTo('screenshots/ios-gateway.png')!@">Here</a> is what this would
|
||||
look like on IOS.
|
||||
|
||||
- Quick Check: At this point, you should already be able to visit an
|
||||
unencrypted HTTP site over the proxy.
|
||||
|
||||
- Open the magic domain <strong>mitm.it</strong> and install the certificate
|
||||
for your device.
|
||||
|
||||
Setting the custom gateway on clients can be automated by serving the settings
|
||||
out to clients over DHCP. This lets set up an interception network where all
|
||||
clients are proxied automatically, which can save time and effort.
|
||||
|
||||
|
||||
<div class="well">
|
||||
<strong style="text-align: center; display: block">Troubleshooting Transparent Mode</strong>
|
||||
<p>Wrong transparent mode configurations are a frequent source of
|
||||
|
||||
<p>Incorrect transparent mode configurations are a frequent source of
|
||||
error. If it doesn't work for you, try the following things:</p>
|
||||
|
||||
<ul>
|
||||
<li>Open mitmproxy's event log (press `e`) - can you spot clientconnect messages?
|
||||
If not, the packets are not arriving at the proxy. A common source is the occurence of ICMP redirects,
|
||||
which means that your machine is telling the client that there's a faster way to the internet by contacting
|
||||
your router directly (see the <em>Transparent Proxying</em> section on how to disable them). If in doubt,
|
||||
<a href="https://wireshark.org/">Wireshark</a> may help you to see whether something arrives at your machine
|
||||
or not.
|
||||
<li>
|
||||
Open mitmproxy's event log (press `e`) - do you see clientconnect
|
||||
messages? If not, the packets are not arriving at the proxy. One common
|
||||
cause is the occurrence of ICMP redirects, which means that your
|
||||
machine is telling the client that there's a faster way to the
|
||||
internet by contacting your router directly (see the
|
||||
<em>Transparent Proxying</em> section on how to disable them). If in
|
||||
doubt, <a href="https://wireshark.org/">Wireshark</a> may help you
|
||||
to see whether something arrives at your machine or not.
|
||||
</li>
|
||||
<li>
|
||||
Have you explicitly configured an HTTP proxy on your device? You do not need mitmproxy's transparent mode
|
||||
then, just start mitmproxy normally. Explicitly setting a proxy and transparent mode contradict each other,
|
||||
settle for one. Do not explicitly redirect traffic to mitmproxy anywhere except for the Gateway setting.
|
||||
Make sure you have not explicitly configured an HTTP proxy on the
|
||||
client. This is not needed in transparent mode.
|
||||
</li>
|
||||
<li>
|
||||
Re-check the instructions in the <em>Transparent Proxying</em> section. Anything you missed?
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
If you encounter any other pitfalls that should be listed here, please let us know!
|
||||
|
||||
</div>
|
||||
|
||||
<h3>(b) Custom Routing</h3>
|
||||
|
||||
Custom routing is a fairly advanced setup which we'll only document briefly here.
|
||||
First and foremost, it usually requires root on your router. The basic idea is to teach your router a custom routing
|
||||
table that says "for requests from ip X, the proxy machine is the next gateway".
|
||||
In some cases, you may need more fine-grained control of which traffic reaches
|
||||
the mitmproxy instance, and which doesn't. You may, for instance, choose only
|
||||
to divert traffic to some hosts into the transparent proxy. There are a huge
|
||||
number of ways to accomplish this, and much will depend on the router or
|
||||
packet filter you're using. In most cases, the configuration will look like
|
||||
this:
|
||||
|
||||
<a href="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
|
||||
<img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@"></a>
|
||||
|
||||
For this setup, we expect you to have a basic understanding of networking in general. In short, you should get started
|
||||
with <a href="@!urlTo('custom-routing.txt')!@">these routing commands</a>. The Troubleshooting part directly above this
|
||||
section might be helpful for you as well.
|
||||
|
||||
<h3>(c) Separate Network</h3>
|
||||
|
||||
Setting up a separate network using a cheap router might be a viable option, too. Such a configuration mostly resembles
|
||||
the idealistic graphic from the beginning (Variant 1). Take a look at the
|
||||
<a href="@!urlTo('tutorials/transparent-dhcp.html')!@">Transparently proxify virtual machines</a> tutorial to see how
|
||||
such a network could be implemented. The troubleshooting section for custom gateways may be helpful for you, too.
|
||||
<img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
|
||||
</a>
|
||||
|
||||
|
||||
<div class="page-header">
|
||||
<h1>Reverse Proxy</h1>
|
||||
</div>
|
||||
|
||||
Mitmproxy is usually used with a client that uses the proxy to access the Internet. Using reverse proxy mode, you can
|
||||
use mitmproxy to represent a server:
|
||||
Mitmproxy is usually used with a client that uses the proxy to access the
|
||||
Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal
|
||||
HTTP server:
|
||||
|
||||
<a href="@!urlTo('schematics/proxy-modes-reverse.png')!@">
|
||||
<img src="@!urlTo('schematics/proxy-modes-reverse.png')!@"></a>
|
||||
<img src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
|
||||
</a>
|
||||
|
||||
There are various use-cases:
|
||||
<ul>
|
||||
<li>
|
||||
Say you have an internal API running at http://example.local/. You could now setup mitmproxy in
|
||||
reverse proxy mode at http://debug.example.local/ and dynamically point clients to this new API endpoint,
|
||||
which provides clients with the same data and you with debug information. Similarly, you could move your real server
|
||||
to a different ip/port and setup mitmproxy at the original place to debug all sessions.
|
||||
</li>
|
||||
<li>
|
||||
Say you're a web developer working on example.com (with a development version running on localhost:8000).
|
||||
You can modify your hosts file so that example.com points to 127.0.0.1 and then run mitmproxy in reverse proxy
|
||||
mode on port 80. You can test your app on the example.com domain and get all requests recorded in mitmproxy.
|
||||
</li>
|
||||
<li>
|
||||
Say you have some toy project that should get SSL support. Simply setup mitmproxy with SSL termination and you're
|
||||
done (<code>mitmdump -p 443 -R https2http://localhost:80/</code>). There are better tools for this specific task (we don't
|
||||
have C performance obviously), but it's definitely a nice and very quick way to setup an SSL-speaking server.
|
||||
</li>
|
||||
<li>
|
||||
Want to add a non-SSL-capable compression proxy in front of your server? You could even spawn a mitmproxy instance
|
||||
that terminates SSL (https2http://...), point it to the compression proxy and let the compression proxy point
|
||||
to a SSL-initiating mitmproxy (http2https://...), which then points to the real server. As you see, it's a fairly
|
||||
flexible thing.
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<p>
|
||||
Please note that cloning Google by using <code>mitmproxy -R http://google.com/</code> does <em>not</em> really work
|
||||
(as in <a href="@!urlTo('screenshots/ios-reverse.png')!@">this screenshot</a>).
|
||||
This may work for the first request, but the HTML remains unchanged: As soon as the user clicks on an non-relative URL
|
||||
(or downloads a non-relative image resource), they speak with Google directly again.
|
||||
</p>
|
||||
<p>
|
||||
On another note, mitmproxy either supports an HTTP or an HTTPS upstream server, not both at the same time. You can
|
||||
simply work around this by spawning a second mitmproxy instance. Each instance listens to one port and talks to one
|
||||
port.
|
||||
</p>
|
||||
- Say you have an internal API running at http://example.local/. You could now
|
||||
set up mitmproxy in reverse proxy mode at http://debug.example.local/ and
|
||||
dynamically point clients to this new API endpoint, which provides clients
|
||||
with the same data and you with debug information. Similarly, you could move
|
||||
your real server to a different IP/port and set up mitmproxy at the original
|
||||
place to debug all sessions.
|
||||
|
||||
- Say you're a web developer working on example.com (with a development
|
||||
version running on localhost:8000). You can modify your hosts file so that
|
||||
example.com points to 127.0.0.1 and then run mitmproxy in reverse proxy mode
|
||||
on port 80. You can test your app on the example.com domain and get all
|
||||
requests recorded in mitmproxy.
|
||||
|
||||
- Say you have some toy project that should get SSL support. Simply set up
|
||||
mitmproxy with SSL termination and you're done (<code>mitmdump -p 443 -R
|
||||
https2http://localhost:80/</code>). There are better tools for this specific
|
||||
task, but mitmproxy is very quick and simple way to set up an SSL-speaking
|
||||
server.
|
||||
|
||||
- Want to add a non-SSL-capable compression proxy in front of your server? You
|
||||
could even spawn a mitmproxy instance that terminates SSL (https2http://...),
|
||||
point it to the compression proxy and let the compression proxy point to a
|
||||
SSL-initiating mitmproxy (http2https://...), which then points to the real
|
||||
server. As you see, it's a fairly flexible thing.
|
||||
|
||||
Note that mitmproxy supports either an HTTP or an HTTPS upstream server, not
|
||||
both at the same time. You can work around this by spawning a second mitmproxy
|
||||
instance.
|
||||
|
||||
<div class="well">
|
||||
<strong style="text-align: center; display: block">Caveat: Interactive Use</strong>
|
||||
|
||||
|
||||
One caveat is that reverse proxy mode is often not sufficient for interactive
|
||||
browsing. Consider trying to clone Google by using:
|
||||
|
||||
<code>mitmproxy -R http://google.com/</code>
|
||||
|
||||
This works for the initial request, but the HTML served to the client remains
|
||||
unchanged. As soon as the user clicks on an non-relative URL (or downloads a
|
||||
non-relative image resource), traffic no longer passes through mitmproxy, and
|
||||
the client connects to Google directly again.
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div class="page-header">
|
||||
<h1>Upstream Proxy</h1>
|
||||
</div>
|
||||
|
||||
<p>
|
||||
If you want to add mitmproxy in front of a different proxy appliance, you can use mitmproxy's upstream mode.
|
||||
In upstream mode, all requests are unconditionally transferred to an upstream proxy or your choice.
|
||||
</p>
|
||||
If you want to chain proxies by adding mitmproxy in front of a different proxy
|
||||
appliance, you can use mitmproxy's upstream mode. In upstream mode, all
|
||||
requests are unconditionally transferred to an upstream proxy of your choice.
|
||||
|
||||
<a href="@!urlTo('schematics/proxy-modes-upstream.png')!@">
|
||||
<img src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
|
||||
|
||||
<p>
|
||||
mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy mode. You could in theory chain multiple
|
||||
mitmproxy instances in a row, but that doesn't make any sense in practice (i.e. outside of our tests).
|
||||
</p>
|
||||
mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy
|
||||
mode. You could in theory chain multiple mitmproxy instances in a row, but
|
||||
that doesn't make any sense in practice (i.e. outside of our tests).
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 16 KiB After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 22 KiB After Width: | Height: | Size: 23 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 17 KiB After Width: | Height: | Size: 14 KiB |
Binary file not shown.
@ -41,10 +41,26 @@ The files created by mitmproxy in the .mitmproxy directory are as follows:
|
||||
Using a custom certificate
|
||||
--------------------------
|
||||
|
||||
You can use your own certificate by passing the __--cert__ option to mitmproxy.
|
||||
You can use your own certificate by passing the <kbd>--cert</kbd> option to mitmproxy. mitmproxy then uses the provided
|
||||
certificate for interception of the specified domains instead of generating a cert signed by its own CA.
|
||||
|
||||
The certificate file is expected to be in the PEM format. You can generate
|
||||
a certificate in this format using these instructions:
|
||||
The certificate file is expected to be in the PEM format.
|
||||
You can include intermediary certificates right below your leaf certificate, so that you PEM file roughly looks like
|
||||
this:
|
||||
|
||||
<pre>
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
<private key>
|
||||
-----END PRIVATE KEY-----
|
||||
-----BEGIN CERTIFICATE-----
|
||||
<cert>
|
||||
-----END CERTIFICATE-----
|
||||
-----BEGIN CERTIFICATE-----
|
||||
<intermediary cert (optional)>
|
||||
-----END CERTIFICATE-----
|
||||
</pre>
|
||||
|
||||
For example, you can generate a certificate in this format using these instructions:
|
||||
|
||||
<pre class="terminal">
|
||||
> openssl genrsa -out cert.key 8192
|
||||
@ -55,6 +71,15 @@ a certificate in this format using these instructions:
|
||||
</pre>
|
||||
|
||||
|
||||
Using a custom certificate authority
|
||||
------------------------------------
|
||||
|
||||
By default, mitmproxy will (generate and) use <samp>~/.mitmproxy/mitmproxy-ca.pem</samp> as the default certificate
|
||||
authority to generate certificates for all domains for which no custom certificate is provided (see above).
|
||||
You can use your own certificate authority by passing the <kbd>--confdir</kbd> option to mitmproxy.
|
||||
mitmproxy will then look for <samp>mitmproxy-ca.pem</samp> in the specified directory. If no such file exists,
|
||||
it will be generated automatically.
|
||||
|
||||
Installing the mitmproxy CA
|
||||
---------------------------
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# This scripts demonstrates how mitmproxy can switch to a different upstream proxy
|
||||
# This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy
|
||||
# in upstream proxy mode.
|
||||
#
|
||||
# Usage: mitmdump -s "change_upstream_proxy.py host"
|
||||
# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s "change_upstream_proxy.py host"
|
||||
from libmproxy.protocol.http import send_connect_request
|
||||
|
||||
alternative_upstream_proxy = ("localhost", 8082)
|
||||
|
||||
@ -36,7 +36,7 @@ class MyMaster(flow.FlowMaster):
|
||||
|
||||
config = proxy.ProxyConfig(
|
||||
port=8080,
|
||||
ca_file=os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
|
||||
cadir="~/.mitmproxy/" # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
|
||||
)
|
||||
state = flow.State()
|
||||
server = ProxyServer(config)
|
||||
|
||||
212
examples/har_extractor.py
Normal file
212
examples/har_extractor.py
Normal file
@ -0,0 +1,212 @@
|
||||
"""
|
||||
This inline script utilizes harparser.HAR from https://github.com/JustusW/harparser
|
||||
to generate a HAR log object.
|
||||
"""
|
||||
try:
|
||||
from harparser import HAR
|
||||
from pytz import UTC
|
||||
except ImportError as e:
|
||||
import sys
|
||||
print >> sys.stderr, "\r\nMissing dependencies: please run `pip install mitmproxy[examples]`.\r\n"
|
||||
raise
|
||||
|
||||
from datetime import datetime, timedelta, tzinfo
|
||||
|
||||
|
||||
class _HARLog(HAR.log):
|
||||
# The attributes need to be registered here for them to actually be available later via self. This is
|
||||
# due to HAREncodable linking __getattr__ to __getitem__. Anything that is set only in __init__ will
|
||||
# just be added as key/value pair to self.__classes__.
|
||||
__page_list__ = []
|
||||
__page_count__ = 0
|
||||
__page_ref__ = {}
|
||||
|
||||
def __init__(self, page_list):
|
||||
self.__page_list__ = page_list
|
||||
self.__page_count__ = 0
|
||||
self.__page_ref__ = {}
|
||||
|
||||
HAR.log.__init__(self, {"version": "1.2",
|
||||
"creator": {"name": "MITMPROXY HARExtractor",
|
||||
"version": "0.1",
|
||||
"comment": ""},
|
||||
"pages": [],
|
||||
"entries": []})
|
||||
|
||||
def reset(self):
|
||||
self.__init__(self.__page_list__)
|
||||
|
||||
def add(self, obj):
|
||||
if isinstance(obj, HAR.pages):
|
||||
self['pages'].append(obj)
|
||||
if isinstance(obj, HAR.entries):
|
||||
self['entries'].append(obj)
|
||||
|
||||
def create_page_id(self):
|
||||
self.__page_count__ += 1
|
||||
return "autopage_%s" % str(self.__page_count__)
|
||||
|
||||
def set_page_ref(self, page, ref):
|
||||
self.__page_ref__[page] = ref
|
||||
|
||||
def get_page_ref(self, page):
|
||||
return self.__page_ref__.get(page, None)
|
||||
|
||||
def get_page_list(self):
|
||||
return self.__page_list__
|
||||
|
||||
|
||||
def start(context, argv):
|
||||
"""
|
||||
On start we create a HARLog instance. You will have to adapt this to suit your actual needs
|
||||
of HAR generation. As it will probably be necessary to cluster logs by IPs or reset them
|
||||
from time to time.
|
||||
"""
|
||||
context.dump_file = None
|
||||
if len(argv) > 1:
|
||||
context.dump_file = argv[1]
|
||||
else:
|
||||
raise ValueError('Usage: -s "har_extractor.py filename" '
|
||||
'(- will output to stdout, filenames ending with .zhar will result in compressed har)')
|
||||
context.HARLog = _HARLog(['https://github.com'])
|
||||
context.seen_server = set()
|
||||
|
||||
|
||||
def response(context, flow):
|
||||
"""
|
||||
Called when a server response has been received. At the time of this message both
|
||||
a request and a response are present and completely done.
|
||||
"""
|
||||
# Values are converted from float seconds to int milliseconds later.
|
||||
ssl_time = -.001
|
||||
connect_time = -.001
|
||||
if flow.server_conn not in context.seen_server:
|
||||
# Calculate the connect_time for this server_conn. Afterwards add it to seen list, in
|
||||
# order to avoid the connect_time being present in entries that use an existing connection.
|
||||
connect_time = flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start
|
||||
context.seen_server.add(flow.server_conn)
|
||||
|
||||
if flow.server_conn.timestamp_ssl_setup is not None:
|
||||
# Get the ssl_time for this server_conn as the difference between the start of the successful
|
||||
# tcp setup and the successful ssl setup. If no ssl setup has been made it is left as -1 since
|
||||
# it doesn't apply to this connection.
|
||||
ssl_time = flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup
|
||||
|
||||
# Calculate the raw timings from the different timestamps present in the request and response object.
|
||||
# For lack of a way to measure it dns timings can not be calculated. The same goes for HAR blocked:
|
||||
# MITMProxy will open a server connection as soon as it receives the host and port from the client
|
||||
# connection. So the time spent waiting is actually spent waiting between request.timestamp_end and
|
||||
# response.timestamp_start thus it correlates to HAR wait instead.
|
||||
timings_raw = {'send': flow.request.timestamp_end - flow.request.timestamp_start,
|
||||
'wait': flow.response.timestamp_start - flow.request.timestamp_end,
|
||||
'receive': flow.response.timestamp_end - flow.response.timestamp_start,
|
||||
'connect': connect_time,
|
||||
'ssl': ssl_time}
|
||||
|
||||
# HAR timings are integers in ms, so we have to re-encode the raw timings to that format.
|
||||
timings = dict([(key, int(1000 * value)) for key, value in timings_raw.iteritems()])
|
||||
|
||||
# The full_time is the sum of all timings. Timings set to -1 will be ignored as per spec.
|
||||
full_time = 0
|
||||
for item in timings.values():
|
||||
if item > -1:
|
||||
full_time += item
|
||||
|
||||
started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, tz=utc).isoformat()
|
||||
|
||||
request_query_string = [{"name": k, "value": v} for k, v in flow.request.get_query()]
|
||||
request_http_version = ".".join([str(v) for v in flow.request.httpversion])
|
||||
# Cookies are shaped as tuples by MITMProxy.
|
||||
request_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.request.get_cookies() or {}).iteritems()]
|
||||
request_headers = [{"name": k, "value": v} for k, v in flow.request.headers]
|
||||
request_headers_size = len(str(flow.request.headers))
|
||||
request_body_size = len(flow.request.content)
|
||||
|
||||
response_http_version = ".".join([str(v) for v in flow.response.httpversion])
|
||||
# Cookies are shaped as tuples by MITMProxy.
|
||||
response_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.response.get_cookies() or {}).iteritems()]
|
||||
response_headers = [{"name": k, "value": v} for k, v in flow.response.headers]
|
||||
response_headers_size = len(str(flow.response.headers))
|
||||
response_body_size = len(flow.response.content)
|
||||
response_body_decoded_size = len(flow.response.get_decoded_content())
|
||||
response_body_compression = response_body_decoded_size - response_body_size
|
||||
response_mime_type = flow.response.headers.get_first('Content-Type', '')
|
||||
response_redirect_url = flow.response.headers.get_first('Location', '')
|
||||
|
||||
entry = HAR.entries({"startedDateTime": started_date_time,
|
||||
"time": full_time,
|
||||
"request": {"method": flow.request.method,
|
||||
"url": flow.request.url,
|
||||
"httpVersion": request_http_version,
|
||||
"cookies": request_cookies,
|
||||
"headers": request_headers,
|
||||
"queryString": request_query_string,
|
||||
"headersSize": request_headers_size,
|
||||
"bodySize": request_body_size, },
|
||||
"response": {"status": flow.response.code,
|
||||
"statusText": flow.response.msg,
|
||||
"httpVersion": response_http_version,
|
||||
"cookies": response_cookies,
|
||||
"headers": response_headers,
|
||||
"content": {"size": response_body_size,
|
||||
"compression": response_body_compression,
|
||||
"mimeType": response_mime_type},
|
||||
"redirectURL": response_redirect_url,
|
||||
"headersSize": response_headers_size,
|
||||
"bodySize": response_body_size, },
|
||||
"cache": {},
|
||||
"timings": timings, })
|
||||
|
||||
# If the current url is in the page list of context.HARLog or does not have a referrer we add it as a new
|
||||
# pages object.
|
||||
if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get('Referer', None) is None:
|
||||
page_id = context.HARLog.create_page_id()
|
||||
context.HARLog.add(HAR.pages({"startedDateTime": entry['startedDateTime'],
|
||||
"id": page_id,
|
||||
"title": flow.request.url, }))
|
||||
context.HARLog.set_page_ref(flow.request.url, page_id)
|
||||
entry['pageref'] = page_id
|
||||
|
||||
# Lookup the referer in the page_ref of context.HARLog to point this entries pageref attribute to the right
|
||||
# pages object, then set it as a new reference to build a reference tree.
|
||||
elif context.HARLog.get_page_ref(flow.request.headers.get('Referer', (None, ))[0]) is not None:
|
||||
entry['pageref'] = context.HARLog.get_page_ref(flow.request.headers['Referer'][0])
|
||||
context.HARLog.set_page_ref(flow.request.headers['Referer'][0], entry['pageref'])
|
||||
|
||||
context.HARLog.add(entry)
|
||||
|
||||
|
||||
def done(context):
|
||||
"""
|
||||
Called once on script shutdown, after any other events.
|
||||
"""
|
||||
from pprint import pprint
|
||||
import json
|
||||
|
||||
json_dump = context.HARLog.json()
|
||||
compressed_json_dump = context.HARLog.compress()
|
||||
|
||||
print "=" * 100
|
||||
if context.dump_file == '-':
|
||||
pprint(json.loads(json_dump))
|
||||
elif context.dump_file.endswith('.zhar'):
|
||||
file(context.dump_file, "w").write(compressed_json_dump)
|
||||
else:
|
||||
file(context.dump_file, "w").write(json_dump)
|
||||
print "=" * 100
|
||||
print "HAR log finished with %s bytes (%s bytes compressed)" % (len(json_dump), len(compressed_json_dump))
|
||||
print "Compression rate is %s%%" % str(100. * len(compressed_json_dump) / len(json_dump))
|
||||
print "=" * 100
|
||||
|
||||
|
||||
def print_attributes(obj, filter_string=None, hide_privates=False):
|
||||
"""
|
||||
Useful helper method to quickly get all attributes of an object and its values.
|
||||
"""
|
||||
for attr in dir(obj):
|
||||
if hide_privates and "__" in attr:
|
||||
continue
|
||||
if filter_string is not None and filter_string not in attr:
|
||||
continue
|
||||
value = getattr(obj, attr)
|
||||
print "%s.%s" % ('obj', attr), value, type(value)
|
||||
34
examples/ignore_websocket.py
Normal file
34
examples/ignore_websocket.py
Normal file
@ -0,0 +1,34 @@
|
||||
# This script makes mitmproxy switch to passthrough mode for all HTTP
|
||||
# responses with "Connection: Upgrade" header. This is useful to make
|
||||
# WebSockets work in untrusted environments.
|
||||
#
|
||||
# Note: Chrome (and possibly other browsers), when explicitly configured
|
||||
# to use a proxy (i.e. mitmproxy's regular mode), send a CONNECT request
|
||||
# to the proxy before they initiate the websocket connection.
|
||||
# To make WebSockets work in these cases, supply
|
||||
# `--ignore :80$` as an additional parameter.
|
||||
# (see http://mitmproxy.org/doc/features/passthrough.html)
|
||||
|
||||
from libmproxy.protocol.http import HTTPRequest
|
||||
from libmproxy.protocol.tcp import TCPHandler
|
||||
from libmproxy.protocol import KILL
|
||||
from libmproxy.script import concurrent
|
||||
|
||||
|
||||
def start(context, argv):
|
||||
HTTPRequest._headers_to_strip_off.remove("Connection")
|
||||
HTTPRequest._headers_to_strip_off.remove("Upgrade")
|
||||
|
||||
|
||||
def done(context):
|
||||
HTTPRequest._headers_to_strip_off.append("Connection")
|
||||
HTTPRequest._headers_to_strip_off.append("Upgrade")
|
||||
|
||||
@concurrent
|
||||
def response(context, flow):
|
||||
if flow.response.headers.get_first("Connection", None) == "Upgrade":
|
||||
# We need to send the response manually now...
|
||||
flow.client_conn.send(flow.response.assemble())
|
||||
# ...and then delegate to tcp passthrough.
|
||||
TCPHandler(flow.live.c, log=False).handle_messages()
|
||||
flow.reply(KILL)
|
||||
@ -36,10 +36,7 @@ class StickyMaster(controller.Master):
|
||||
flow.reply()
|
||||
|
||||
|
||||
config = proxy.ProxyConfig(
|
||||
port=8080,
|
||||
ca_file=os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
|
||||
)
|
||||
config = proxy.ProxyConfig(port=8080)
|
||||
server = ProxyServer(config)
|
||||
m = StickyMaster(server)
|
||||
m.run()
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
from __future__ import absolute_import
|
||||
import os
|
||||
import re
|
||||
import argparse
|
||||
from argparse import ArgumentTypeError
|
||||
import configargparse
|
||||
from netlib import http
|
||||
from . import filt, utils
|
||||
from . import filt, utils, version
|
||||
from .proxy import config
|
||||
|
||||
APP_HOST = "mitm.it"
|
||||
@ -23,7 +23,9 @@ def _parse_hook(s):
|
||||
elif len(parts) == 3:
|
||||
patt, a, b = parts
|
||||
else:
|
||||
raise ParseException("Malformed hook specifier - too few clauses: %s" % s)
|
||||
raise ParseException(
|
||||
"Malformed hook specifier - too few clauses: %s" % s
|
||||
)
|
||||
|
||||
if not a:
|
||||
raise ParseException("Empty clause: %s" % str(patt))
|
||||
@ -102,7 +104,9 @@ def parse_server_spec(url):
|
||||
|
||||
p = http.parse_url(normalized_url)
|
||||
if not p or not p[1]:
|
||||
raise ArgumentTypeError("Invalid server specification: %s" % url)
|
||||
raise configargparse.ArgumentTypeError(
|
||||
"Invalid server specification: %s" % url
|
||||
)
|
||||
|
||||
if url.lower().startswith("https2http"):
|
||||
ssl = [True, False]
|
||||
@ -131,17 +135,19 @@ def get_common_options(options):
|
||||
try:
|
||||
p = parse_replace_hook(i)
|
||||
except ParseException, e:
|
||||
raise ArgumentTypeError(e.message)
|
||||
raise configargparse.ArgumentTypeError(e.message)
|
||||
reps.append(p)
|
||||
for i in options.replace_file:
|
||||
try:
|
||||
patt, rex, path = parse_replace_hook(i)
|
||||
except ParseException, e:
|
||||
raise ArgumentTypeError(e.message)
|
||||
raise configargparse.ArgumentTypeError(e.message)
|
||||
try:
|
||||
v = open(path, "rb").read()
|
||||
except IOError, e:
|
||||
raise ArgumentTypeError("Could not read replace file: %s" % path)
|
||||
raise configargparse.ArgumentTypeError(
|
||||
"Could not read replace file: %s" % path
|
||||
)
|
||||
reps.append((patt, rex, v))
|
||||
|
||||
setheaders = []
|
||||
@ -149,7 +155,7 @@ def get_common_options(options):
|
||||
try:
|
||||
p = parse_setheader(i)
|
||||
except ParseException, e:
|
||||
raise ArgumentTypeError(e.message)
|
||||
raise configargparse.ArgumentTypeError(e.message)
|
||||
setheaders.append(p)
|
||||
|
||||
return dict(
|
||||
@ -183,14 +189,23 @@ def get_common_options(options):
|
||||
|
||||
def common_options(parser):
|
||||
parser.add_argument(
|
||||
"--anticache",
|
||||
action="store_true", dest="anticache", default=False,
|
||||
help="Strip out request headers that might cause the server to return 304-not-modified."
|
||||
'--version',
|
||||
action= 'version',
|
||||
version= "%(prog)s" + " " + version.VERSION
|
||||
)
|
||||
parser.add_argument(
|
||||
"--confdir",
|
||||
action="store", type=str, dest="confdir", default='~/.mitmproxy',
|
||||
help="Configuration directory. (~/.mitmproxy)"
|
||||
"--anticache",
|
||||
action="store_true", dest="anticache", default=False,
|
||||
|
||||
help="""
|
||||
Strip out request headers that might cause the server to return
|
||||
304-not-modified.
|
||||
"""
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cadir",
|
||||
action="store", type=str, dest="cadir", default=config.CA_DIR,
|
||||
help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
@ -198,112 +213,150 @@ def common_options(parser):
|
||||
help="Use the Host header to construct URLs for display."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-q",
|
||||
"-q", "--quiet",
|
||||
action="store_true", dest="quiet",
|
||||
help="Quiet."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-r",
|
||||
"-r", "--read-flows",
|
||||
action="store", dest="rfile", default=None,
|
||||
help="Read flows from file."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-s",
|
||||
"-s", "--script",
|
||||
action="append", type=str, dest="scripts", default=[],
|
||||
metavar='"script.py --bar"',
|
||||
help="Run a script. Surround with quotes to pass script arguments. Can be passed multiple times."
|
||||
help="""
|
||||
Run a script. Surround with quotes to pass script arguments. Can be
|
||||
passed multiple times.
|
||||
"""
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
action="store", dest="stickycookie_filt", default=None, metavar="FILTER",
|
||||
"-t", "--stickycookie",
|
||||
action="store",
|
||||
dest="stickycookie_filt",
|
||||
default=None,
|
||||
metavar="FILTER",
|
||||
help="Set sticky cookie filter. Matched against requests."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-u",
|
||||
"-u", "--stickyauth",
|
||||
action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
|
||||
help="Set sticky auth filter. Matched against requests."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"-v", "--verbose",
|
||||
action="store_const", dest="verbose", default=1, const=2,
|
||||
help="Increase event log verbosity."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-w",
|
||||
"-w", "--wfile",
|
||||
action="store", dest="wfile", default=None,
|
||||
help="Write flows to file."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-z",
|
||||
"-z", "--anticomp",
|
||||
action="store_true", dest="anticomp", default=False,
|
||||
help="Try to convince servers to send us un-compressed data."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-Z",
|
||||
"-Z", "--body-size-limit",
|
||||
action="store", dest="body_size_limit", default=None,
|
||||
metavar="SIZE",
|
||||
help="Byte size limit of HTTP request and response bodies." \
|
||||
help="Byte size limit of HTTP request and response bodies."
|
||||
" Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stream",
|
||||
action="store", dest="stream_large_bodies", default=None,
|
||||
metavar="SIZE",
|
||||
help="Stream data to the client if response body exceeds the given threshold. "
|
||||
"If streamed, the body will not be stored in any way. Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
|
||||
help="""
|
||||
Stream data to the client if response body exceeds the given
|
||||
threshold. If streamed, the body will not be stored in any way.
|
||||
Understands k/m/g suffixes, i.e. 3m for 3 megabytes.
|
||||
"""
|
||||
)
|
||||
|
||||
group = parser.add_argument_group("Proxy Options")
|
||||
# We could make a mutually exclusive group out of -R, -U, -T, but we don't do that because
|
||||
# - --upstream-server should be in that group as well, but it's already in a different group.
|
||||
# - our own error messages are more helpful
|
||||
# We could make a mutually exclusive group out of -R, -U, -T, but we don't
|
||||
# do that because - --upstream-server should be in that group as well, but
|
||||
# it's already in a different group. - our own error messages are more
|
||||
# helpful
|
||||
group.add_argument(
|
||||
"-b",
|
||||
"-b", "--bind-address",
|
||||
action="store", type=str, dest="addr", default='',
|
||||
help="Address to bind proxy to (defaults to all interfaces)"
|
||||
)
|
||||
group.add_argument(
|
||||
"-I", "--ignore",
|
||||
action="append", type=str, dest="ignore", default=[],
|
||||
action="append", type=str, dest="ignore_hosts", default=[],
|
||||
metavar="HOST",
|
||||
help="Ignore host and forward all traffic without processing it. "
|
||||
"In transparent mode, it is recommended to use an IP address (range), not the hostname. "
|
||||
"In regular mode, only SSL traffic is ignored and the hostname should be used. "
|
||||
"The supplied value is interpreted as a regular expression and matched on the ip or the hostname. "
|
||||
"Can be passed multiple times. "
|
||||
help="""
|
||||
Ignore host and forward all traffic without processing it. In
|
||||
transparent mode, it is recommended to use an IP address (range),
|
||||
not the hostname. In regular mode, only SSL traffic is ignored and
|
||||
the hostname should be used. The supplied value is interpreted as a
|
||||
regular expression and matched on the ip or the hostname. Can be
|
||||
passed multiple times.
|
||||
"""
|
||||
)
|
||||
group.add_argument(
|
||||
"-n",
|
||||
"--tcp",
|
||||
action="append", type=str, dest="tcp_hosts", default=[],
|
||||
metavar="HOST",
|
||||
help="""
|
||||
Generic TCP SSL proxy mode for all hosts that match the pattern.
|
||||
Similar to --ignore, but SSL connections are intercepted. The
|
||||
communication contents are printed to the event log in verbose mode.
|
||||
"""
|
||||
)
|
||||
group.add_argument(
|
||||
"-n", "--no-server",
|
||||
action="store_true", dest="no_server",
|
||||
help="Don't start a proxy server."
|
||||
)
|
||||
group.add_argument(
|
||||
"-p",
|
||||
"-p", "--port",
|
||||
action="store", type=int, dest="port", default=8080,
|
||||
help="Proxy service port."
|
||||
)
|
||||
group.add_argument(
|
||||
"-R",
|
||||
action="store", type=parse_server_spec, dest="reverse_proxy", default=None,
|
||||
help="Forward all requests to upstream HTTP server: http[s][2http[s]]://host[:port]"
|
||||
"-R", "--reverse",
|
||||
action="store",
|
||||
type=parse_server_spec,
|
||||
dest="reverse_proxy",
|
||||
default=None,
|
||||
help="""
|
||||
Forward all requests to upstream HTTP server:
|
||||
http[s][2http[s]]://host[:port]
|
||||
"""
|
||||
)
|
||||
group.add_argument(
|
||||
"-T",
|
||||
"--socks",
|
||||
action="store_true", dest="socks_proxy", default=False,
|
||||
help="Set SOCKS5 proxy mode."
|
||||
)
|
||||
group.add_argument(
|
||||
"-T", "--transparent",
|
||||
action="store_true", dest="transparent_proxy", default=False,
|
||||
help="Set transparent proxy mode."
|
||||
)
|
||||
group.add_argument(
|
||||
"-U",
|
||||
action="store", type=parse_server_spec, dest="upstream_proxy", default=None,
|
||||
"-U", "--upstream",
|
||||
action="store",
|
||||
type=parse_server_spec,
|
||||
dest="upstream_proxy",
|
||||
default=None,
|
||||
help="Forward all requests to upstream proxy server: http://host[:port]"
|
||||
)
|
||||
|
||||
group = parser.add_argument_group(
|
||||
"Advanced Proxy Options",
|
||||
"""
|
||||
The following options allow a custom adjustment of the proxy behavior.
|
||||
Normally, you don't want to use these options directly and use the provided wrappers instead (-R, -U, -T).
|
||||
""".strip()
|
||||
The following options allow a custom adjustment of the proxy
|
||||
behavior. Normally, you don't want to use these options directly and
|
||||
use the provided wrappers instead (-R, -U, -T).
|
||||
"""
|
||||
)
|
||||
group.add_argument(
|
||||
"--http-form-in", dest="http_form_in", default=None,
|
||||
@ -318,38 +371,44 @@ def common_options(parser):
|
||||
|
||||
group = parser.add_argument_group("Onboarding App")
|
||||
group.add_argument(
|
||||
"-a",
|
||||
"-a", "--noapp",
|
||||
action="store_false", dest="app", default=True,
|
||||
help="Disable the mitmproxy onboarding app."
|
||||
)
|
||||
group.add_argument(
|
||||
"--app-host",
|
||||
action="store", dest="app_host", default=APP_HOST, metavar="host",
|
||||
help="Domain to serve the onboarding app from. For transparent mode, use an IP when\
|
||||
a DNS entry for the app domain is not present. Default: %s" % APP_HOST
|
||||
|
||||
help="""
|
||||
Domain to serve the onboarding app from. For transparent mode, use
|
||||
an IP when a DNS entry for the app domain is not present. Default:
|
||||
%s
|
||||
""" % APP_HOST
|
||||
)
|
||||
group.add_argument(
|
||||
"--app-port",
|
||||
action="store", dest="app_port", default=APP_PORT, type=int, metavar="80",
|
||||
action="store",
|
||||
dest="app_port",
|
||||
default=APP_PORT,
|
||||
type=int,
|
||||
metavar="80",
|
||||
help="Port to serve the onboarding app from."
|
||||
)
|
||||
|
||||
group = parser.add_argument_group("Client Replay")
|
||||
group.add_argument(
|
||||
"-c",
|
||||
"-c", "--client-replay",
|
||||
action="store", dest="client_replay", default=None, metavar="PATH",
|
||||
help="Replay client requests from a saved file."
|
||||
)
|
||||
|
||||
group = parser.add_argument_group("Server Replay")
|
||||
group.add_argument(
|
||||
"-S",
|
||||
"-S", "--server-replay",
|
||||
action="store", dest="server_replay", default=None, metavar="PATH",
|
||||
help="Replay server responses from a saved file."
|
||||
)
|
||||
group.add_argument(
|
||||
"-k",
|
||||
"-k", "--kill",
|
||||
action="store_true", dest="kill", default=False,
|
||||
help="Kill extra requests during replay."
|
||||
)
|
||||
@ -362,8 +421,10 @@ def common_options(parser):
|
||||
group.add_argument(
|
||||
"--norefresh",
|
||||
action="store_true", dest="norefresh", default=False,
|
||||
help="Disable response refresh, "
|
||||
"which updates times in cookies and headers for replayed responses."
|
||||
help="""
|
||||
Disable response refresh, which updates times in cookies and headers
|
||||
for replayed responses.
|
||||
"""
|
||||
)
|
||||
group.add_argument(
|
||||
"--no-pop",
|
||||
@ -374,14 +435,18 @@ def common_options(parser):
|
||||
group.add_argument(
|
||||
"--replay-ignore-content",
|
||||
action="store_true", dest="replay_ignore_content", default=False,
|
||||
help="Ignore request's content while searching for a saved flow to replay"
|
||||
help="""
|
||||
Ignore request's content while searching for a saved flow to replay
|
||||
"""
|
||||
)
|
||||
group.add_argument(
|
||||
"--replay-ignore-param",
|
||||
action="append", dest="replay_ignore_params", type=str,
|
||||
help="Request's parameters to be ignored while searching for a saved flow to replay"
|
||||
"Can be passed multiple times."
|
||||
)
|
||||
help="""
|
||||
Request's parameters to be ignored while searching for a saved flow
|
||||
to replay. Can be passed multiple times.
|
||||
"""
|
||||
)
|
||||
|
||||
group = parser.add_argument_group(
|
||||
"Replacements",
|
||||
@ -399,9 +464,12 @@ def common_options(parser):
|
||||
)
|
||||
group.add_argument(
|
||||
"--replace-from-file",
|
||||
action="append", type=str, dest="replace_file", default=[],
|
||||
metavar="PATH",
|
||||
help="Replacement pattern, where the replacement clause is a path to a file."
|
||||
action = "append", type=str, dest="replace_file", default=[],
|
||||
metavar = "PATH",
|
||||
help = """
|
||||
Replacement pattern, where the replacement clause is a path to a
|
||||
file.
|
||||
"""
|
||||
)
|
||||
|
||||
group = parser.add_argument_group(
|
||||
@ -437,7 +505,10 @@ def common_options(parser):
|
||||
"--singleuser",
|
||||
action="store", dest="auth_singleuser", type=str,
|
||||
metavar="USER",
|
||||
help="Allows access to a a single user, specified in the form username:password."
|
||||
help="""
|
||||
Allows access to a a single user, specified in the form
|
||||
username:password.
|
||||
"""
|
||||
)
|
||||
user_specification_group.add_argument(
|
||||
"--htpasswd",
|
||||
@ -447,3 +518,116 @@ def common_options(parser):
|
||||
)
|
||||
|
||||
config.ssl_option_group(parser)
|
||||
|
||||
|
||||
def mitmproxy():
|
||||
# Don't import libmproxy.console for mitmdump, urwid is not available on all
|
||||
# platforms.
|
||||
from .console import palettes
|
||||
|
||||
parser = configargparse.ArgumentParser(
|
||||
usage="%(prog)s [options]",
|
||||
args_for_setting_config_path = ["--conf"],
|
||||
default_config_files = [
|
||||
os.path.join(config.CA_DIR, "common.conf"),
|
||||
os.path.join(config.CA_DIR, "mitmproxy.conf")
|
||||
],
|
||||
add_config_file_help = True,
|
||||
add_env_var_help = True
|
||||
)
|
||||
common_options(parser)
|
||||
parser.add_argument(
|
||||
"--palette", type=str, default="dark",
|
||||
action="store", dest="palette",
|
||||
help="Select color palette: " + ", ".join(palettes.palettes.keys())
|
||||
)
|
||||
parser.add_argument(
|
||||
"-e", "--eventlog",
|
||||
action="store_true", dest="eventlog",
|
||||
help="Show event log."
|
||||
)
|
||||
group = parser.add_argument_group(
|
||||
"Filters",
|
||||
"See help in mitmproxy for filter expression syntax."
|
||||
)
|
||||
group.add_argument(
|
||||
"-i", "--intercept", action="store",
|
||||
type=str, dest="intercept", default=None,
|
||||
help="Intercept filter expression."
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def mitmdump():
|
||||
parser = configargparse.ArgumentParser(
|
||||
usage="%(prog)s [options] [filter]",
|
||||
args_for_setting_config_path = ["--conf"],
|
||||
default_config_files = [
|
||||
os.path.join(config.CA_DIR, "common.conf"),
|
||||
os.path.join(config.CA_DIR, "mitmdump.conf")
|
||||
],
|
||||
add_config_file_help = True,
|
||||
add_env_var_help = True
|
||||
)
|
||||
|
||||
common_options(parser)
|
||||
parser.add_argument(
|
||||
"--keepserving",
|
||||
action= "store_true", dest="keepserving", default=False,
|
||||
help= """
|
||||
Continue serving after client playback or file read. We exit by
|
||||
default.
|
||||
"""
|
||||
)
|
||||
parser.add_argument(
|
||||
"-d", "--detail",
|
||||
action="count", dest="flow_detail", default=1,
|
||||
help="Increase flow detail display level. Can be passed multiple times."
|
||||
)
|
||||
parser.add_argument('args', nargs="...")
|
||||
return parser
|
||||
|
||||
|
||||
def mitmweb():
|
||||
parser = configargparse.ArgumentParser(
|
||||
usage="%(prog)s [options]",
|
||||
args_for_setting_config_path = ["--conf"],
|
||||
default_config_files = [
|
||||
os.path.join(config.CA_DIR, "common.conf"),
|
||||
os.path.join(config.CA_DIR, "mitmweb.conf")
|
||||
],
|
||||
add_config_file_help = True,
|
||||
add_env_var_help = True
|
||||
)
|
||||
|
||||
group = parser.add_argument_group("Mitmweb")
|
||||
group.add_argument(
|
||||
"--wport",
|
||||
action="store", type=int, dest="wport", default=8081,
|
||||
metavar="PORT",
|
||||
help="Mitmweb port."
|
||||
)
|
||||
group.add_argument(
|
||||
"--wiface",
|
||||
action="store", dest="wiface", default="127.0.0.1",
|
||||
metavar="IFACE",
|
||||
help="Mitmweb interface."
|
||||
)
|
||||
group.add_argument(
|
||||
"--wdebug",
|
||||
action="store_true", dest="wdebug",
|
||||
help="Turn on mitmweb debugging"
|
||||
)
|
||||
|
||||
common_options(parser)
|
||||
group = parser.add_argument_group(
|
||||
"Filters",
|
||||
"See help in mitmproxy for filter expression syntax."
|
||||
)
|
||||
group.add_argument(
|
||||
"-i", "--intercept", action="store",
|
||||
type=str, dest="intercept", default=None,
|
||||
help="Intercept filter expression."
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
@ -129,10 +129,14 @@ class StatusBar(common.WWrap):
|
||||
r.append(":%s in file]"%self.master.server_playback.count())
|
||||
else:
|
||||
r.append(":%s to go]"%self.master.server_playback.count())
|
||||
if self.master.get_ignore():
|
||||
if self.master.get_ignore_filter():
|
||||
r.append("[")
|
||||
r.append(("heading_key", "I"))
|
||||
r.append("gnore:%d]"%len(self.master.get_ignore()))
|
||||
r.append("gnore:%d]" % len(self.master.get_ignore_filter()))
|
||||
if self.master.get_tcp_filter():
|
||||
r.append("[")
|
||||
r.append(("heading_key", "T"))
|
||||
r.append("CP:%d]" % len(self.master.get_tcp_filter()))
|
||||
if self.master.state.intercept_txt:
|
||||
r.append("[")
|
||||
r.append(("heading_key", "i"))
|
||||
@ -512,7 +516,8 @@ class ConsoleMaster(flow.FlowMaster):
|
||||
self.start_server_playback(
|
||||
ret,
|
||||
self.killextra, self.rheaders,
|
||||
False, self.nopop
|
||||
False, self.nopop,
|
||||
self.options.replay_ignore_params, self.options.replay_ignore_content
|
||||
)
|
||||
|
||||
def spawn_editor(self, data):
|
||||
@ -798,9 +803,13 @@ class ConsoleMaster(flow.FlowMaster):
|
||||
for command in commands:
|
||||
self.load_script(command)
|
||||
|
||||
def edit_ignore(self, ignore):
|
||||
def edit_ignore_filter(self, ignore):
|
||||
patterns = (x[0] for x in ignore)
|
||||
self.set_ignore(patterns)
|
||||
self.set_ignore_filter(patterns)
|
||||
|
||||
def edit_tcp_filter(self, tcp):
|
||||
patterns = (x[0] for x in tcp)
|
||||
self.set_tcp_filter(patterns)
|
||||
|
||||
def loop(self):
|
||||
changed = True
|
||||
@ -811,7 +820,7 @@ class ConsoleMaster(flow.FlowMaster):
|
||||
self.statusbar.redraw()
|
||||
size = self.drawscreen()
|
||||
changed = self.tick(self.masterq, 0.01)
|
||||
self.ui.set_input_timeouts(max_wait=0.1)
|
||||
self.ui.set_input_timeouts(max_wait=0.01)
|
||||
keys = self.ui.get_input()
|
||||
if keys:
|
||||
changed = True
|
||||
@ -860,10 +869,18 @@ class ConsoleMaster(flow.FlowMaster):
|
||||
)
|
||||
elif k == "I":
|
||||
self.view_grideditor(
|
||||
grideditor.IgnoreEditor(
|
||||
grideditor.HostPatternEditor(
|
||||
self,
|
||||
[[x] for x in self.get_ignore()],
|
||||
self.edit_ignore
|
||||
[[x] for x in self.get_ignore_filter()],
|
||||
self.edit_ignore_filter
|
||||
)
|
||||
)
|
||||
elif k == "T":
|
||||
self.view_grideditor(
|
||||
grideditor.HostPatternEditor(
|
||||
self,
|
||||
[[x] for x in self.get_tcp_filter()],
|
||||
self.edit_tcp_filter
|
||||
)
|
||||
)
|
||||
elif k == "i":
|
||||
@ -1033,7 +1050,7 @@ class ConsoleMaster(flow.FlowMaster):
|
||||
self.eventlist[:] = []
|
||||
|
||||
def add_event(self, e, level="info"):
|
||||
needed = dict(error=1, info=1, debug=2).get(level, 1)
|
||||
needed = dict(error=0, info=1, debug=2).get(level, 1)
|
||||
if self.options.verbosity < needed:
|
||||
return
|
||||
|
||||
|
||||
@ -120,13 +120,15 @@ class ConnectionItem(common.WWrap):
|
||||
self.master.start_server_playback(
|
||||
[i.copy() for i in self.master.state.view],
|
||||
self.master.killextra, self.master.rheaders,
|
||||
False, self.master.nopop
|
||||
False, self.master.nopop,
|
||||
self.master.options.replay_ignore_params, self.master.options.replay_ignore_content
|
||||
)
|
||||
elif k == "t":
|
||||
self.master.start_server_playback(
|
||||
[self.flow.copy()],
|
||||
self.master.killextra, self.master.rheaders,
|
||||
False, self.master.nopop
|
||||
False, self.master.nopop,
|
||||
self.master.options.replay_ignore_params, self.master.options.replay_ignore_content
|
||||
)
|
||||
else:
|
||||
self.master.path_prompt(
|
||||
|
||||
@ -574,9 +574,8 @@ class FlowView(common.WWrap):
|
||||
else:
|
||||
if not self.flow.response:
|
||||
self.flow.response = HTTPResponse(
|
||||
self.flow.request,
|
||||
self.flow.request.httpversion,
|
||||
200, "OK", flow.ODictCaseless(), "", None
|
||||
200, "OK", flow.ODictCaseless(), ""
|
||||
)
|
||||
self.flow.response.reply = controller.DummyReply()
|
||||
conn = self.flow.response
|
||||
@ -749,7 +748,7 @@ class FlowView(common.WWrap):
|
||||
self.master.statusbar.message("")
|
||||
elif key == "m":
|
||||
p = list(contentview.view_prompts)
|
||||
p.insert(0, ("clear", "c"))
|
||||
p.insert(0, ("Clear", "C"))
|
||||
self.master.prompt_onekey(
|
||||
"Display mode",
|
||||
p,
|
||||
|
||||
@ -123,12 +123,13 @@ class GridWalker(urwid.ListWalker):
|
||||
except ValueError:
|
||||
self.editor.master.statusbar.message("Invalid Python-style string encoding.", 1000)
|
||||
return
|
||||
|
||||
errors = self.lst[self.focus][1]
|
||||
emsg = self.editor.is_error(self.focus_col, val)
|
||||
if emsg:
|
||||
self.editor.master.statusbar.message(emsg, 1000)
|
||||
errors.add(self.focus_col)
|
||||
else:
|
||||
errors.discard(self.focus_col)
|
||||
|
||||
row = list(self.lst[self.focus][0])
|
||||
row[self.focus_col] = val
|
||||
@ -320,9 +321,11 @@ class GridEditor(common.WWrap):
|
||||
elif key == "d":
|
||||
self.walker.delete_focus()
|
||||
elif key == "r":
|
||||
self.master.path_prompt("Read file: ", "", self.read_file)
|
||||
if self.walker.get_current_value() is not None:
|
||||
self.master.path_prompt("Read file: ", "", self.read_file)
|
||||
elif key == "R":
|
||||
self.master.path_prompt("Read unescaped file: ", "", self.read_file, True)
|
||||
if self.walker.get_current_value() is not None:
|
||||
self.master.path_prompt("Read unescaped file: ", "", self.read_file, True)
|
||||
elif key == "e":
|
||||
o = self.walker.get_current_value()
|
||||
if o is not None:
|
||||
@ -495,8 +498,8 @@ class ScriptEditor(GridEditor):
|
||||
return str(v)
|
||||
|
||||
|
||||
class IgnoreEditor(GridEditor):
|
||||
title = "Editing ignore patterns"
|
||||
class HostPatternEditor(GridEditor):
|
||||
title = "Editing host patterns"
|
||||
columns = 1
|
||||
headings = ("Regex (matched on hostname:port / ip:port)",)
|
||||
|
||||
|
||||
@ -119,6 +119,7 @@ class HelpView(urwid.ListBox):
|
||||
("s", "add/remove scripts"),
|
||||
("S", "server replay"),
|
||||
("t", "set sticky cookie expression"),
|
||||
("T", "set tcp proxying pattern"),
|
||||
("u", "set sticky auth expression"),
|
||||
]
|
||||
text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
|
||||
|
||||
@ -1,10 +1,13 @@
|
||||
from __future__ import absolute_import
|
||||
import sys, os
|
||||
import sys
|
||||
import os
|
||||
import netlib.utils
|
||||
from . import flow, filt, utils
|
||||
from .protocol import http
|
||||
|
||||
class DumpError(Exception): pass
|
||||
|
||||
class DumpError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Options(object):
|
||||
@ -37,6 +40,7 @@ class Options(object):
|
||||
"replay_ignore_content",
|
||||
"replay_ignore_params",
|
||||
]
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
for k, v in kwargs.items():
|
||||
setattr(self, k, v)
|
||||
@ -71,7 +75,7 @@ class DumpMaster(flow.FlowMaster):
|
||||
self.anticache = options.anticache
|
||||
self.anticomp = options.anticomp
|
||||
self.showhost = options.showhost
|
||||
self.replay_ignore_params = options.replay_ignore_params
|
||||
self.replay_ignore_params = options.replay_ignore_params
|
||||
self.replay_ignore_content = options.replay_ignore_content
|
||||
self.refresh_server_playback = options.refresh_server_playback
|
||||
|
||||
@ -88,7 +92,6 @@ class DumpMaster(flow.FlowMaster):
|
||||
if options.stickyauth:
|
||||
self.set_stickyauth(options.stickyauth)
|
||||
|
||||
|
||||
if options.wfile:
|
||||
path = os.path.expanduser(options.wfile)
|
||||
try:
|
||||
@ -152,7 +155,7 @@ class DumpMaster(flow.FlowMaster):
|
||||
return flows
|
||||
|
||||
def add_event(self, e, level="info"):
|
||||
needed = dict(error=1, info=1, debug=2).get(level, 1)
|
||||
needed = dict(error=0, info=1, debug=2).get(level, 1)
|
||||
if self.o.verbosity >= needed:
|
||||
print >> self.outfile, e
|
||||
self.outfile.flush()
|
||||
@ -202,7 +205,7 @@ class DumpMaster(flow.FlowMaster):
|
||||
elif self.o.flow_detail >= 3:
|
||||
print >> self.outfile, str_request(f, self.showhost)
|
||||
print >> self.outfile, self.indent(4, f.request.headers)
|
||||
if utils.isBin(f.request.content):
|
||||
if f.request.content != http.CONTENT_MISSING and utils.isBin(f.request.content):
|
||||
d = netlib.utils.hexdump(f.request.content)
|
||||
d = "\n".join("%s\t%s %s"%i for i in d)
|
||||
print >> self.outfile, self.indent(4, d)
|
||||
|
||||
@ -343,7 +343,9 @@ bnf = _make()
|
||||
|
||||
def parse(s):
|
||||
try:
|
||||
return bnf.parseString(s, parseAll=True)[0]
|
||||
filt = bnf.parseString(s, parseAll=True)[0]
|
||||
filt.pattern = s
|
||||
return filt
|
||||
except pp.ParseException:
|
||||
return None
|
||||
except ValueError:
|
||||
|
||||
@ -11,7 +11,7 @@ import netlib.http
|
||||
from . import controller, protocol, tnetstring, filt, script, version
|
||||
from .onboarding import app
|
||||
from .protocol import http, handle
|
||||
from .proxy.config import parse_host_pattern
|
||||
from .proxy.config import HostMatcher
|
||||
import urlparse
|
||||
|
||||
ODict = odict.ODict
|
||||
@ -27,7 +27,12 @@ class AppRegistry:
|
||||
Add a WSGI app to the registry, to be served for requests to the
|
||||
specified domain, on the specified port.
|
||||
"""
|
||||
self.apps[(domain, port)] = wsgi.WSGIAdaptor(app, domain, port, version.NAMEVERSION)
|
||||
self.apps[(domain, port)] = wsgi.WSGIAdaptor(
|
||||
app,
|
||||
domain,
|
||||
port,
|
||||
version.NAMEVERSION
|
||||
)
|
||||
|
||||
def get(self, request):
|
||||
"""
|
||||
@ -72,7 +77,8 @@ class ReplaceHooks:
|
||||
|
||||
def get_specs(self):
|
||||
"""
|
||||
Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) tuples.
|
||||
Retrieve the hook specifcations. Returns a list of (fpatt, rex, s)
|
||||
tuples.
|
||||
"""
|
||||
return [i[:3] for i in self.lst]
|
||||
|
||||
@ -119,7 +125,8 @@ class SetHeaders:
|
||||
|
||||
def get_specs(self):
|
||||
"""
|
||||
Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) tuples.
|
||||
Retrieve the hook specifcations. Returns a list of (fpatt, rex, s)
|
||||
tuples.
|
||||
"""
|
||||
return [i[:3] for i in self.lst]
|
||||
|
||||
@ -162,6 +169,7 @@ class ClientPlaybackState:
|
||||
def __init__(self, flows, exit):
|
||||
self.flows, self.exit = flows, exit
|
||||
self.current = None
|
||||
self.testing = False # Disables actual replay for testing.
|
||||
|
||||
def count(self):
|
||||
return len(self.flows)
|
||||
@ -179,18 +187,16 @@ class ClientPlaybackState:
|
||||
if flow is self.current:
|
||||
self.current = None
|
||||
|
||||
def tick(self, master, testing=False):
|
||||
"""
|
||||
testing: Disables actual replay for testing.
|
||||
"""
|
||||
def tick(self, master):
|
||||
if self.flows and not self.current:
|
||||
n = self.flows.pop(0)
|
||||
n.reply = controller.DummyReply()
|
||||
self.current = master.handle_request(n)
|
||||
if not testing and not self.current.response:
|
||||
master.replay_request(self.current) # pragma: no cover
|
||||
elif self.current.response:
|
||||
master.handle_response(self.current)
|
||||
self.current = self.flows.pop(0).copy()
|
||||
if not self.testing:
|
||||
master.replay_request(self.current)
|
||||
else:
|
||||
self.current.reply = controller.DummyReply()
|
||||
master.handle_request(self.current)
|
||||
if self.current.response:
|
||||
master.handle_response(self.current)
|
||||
|
||||
|
||||
class ServerPlaybackState:
|
||||
@ -219,9 +225,10 @@ class ServerPlaybackState:
|
||||
queriesArray = urlparse.parse_qsl(query)
|
||||
|
||||
filtered = []
|
||||
ignore_params = self.ignore_params or []
|
||||
for p in queriesArray:
|
||||
if p[0] not in self.ignore_params:
|
||||
filtered.append(p)
|
||||
if p[0] not in ignore_params:
|
||||
filtered.append(p)
|
||||
|
||||
key = [
|
||||
str(r.host),
|
||||
@ -339,11 +346,13 @@ class State(object):
|
||||
# These are compiled filt expressions:
|
||||
self._limit = None
|
||||
self.intercept = None
|
||||
self._limit_txt = None
|
||||
|
||||
@property
|
||||
def limit_txt(self):
|
||||
return self._limit_txt
|
||||
if self._limit:
|
||||
return self._limit.pattern
|
||||
else:
|
||||
return None
|
||||
|
||||
def flow_count(self):
|
||||
return len(self._flow_list)
|
||||
@ -362,6 +371,8 @@ class State(object):
|
||||
"""
|
||||
Add a request to the state. Returns the matching flow.
|
||||
"""
|
||||
if flow in self._flow_list: # catch flow replay
|
||||
return flow
|
||||
self._flow_list.append(flow)
|
||||
if flow.match(self._limit):
|
||||
self.view.append(flow)
|
||||
@ -398,10 +409,8 @@ class State(object):
|
||||
if not f:
|
||||
return "Invalid filter expression."
|
||||
self._limit = f
|
||||
self._limit_txt = txt
|
||||
else:
|
||||
self._limit = None
|
||||
self._limit_txt = None
|
||||
self.recalculate_view()
|
||||
|
||||
def set_intercept(self, txt):
|
||||
@ -465,7 +474,7 @@ class FlowMaster(controller.Master):
|
||||
self.refresh_server_playback = False
|
||||
self.replacehooks = ReplaceHooks()
|
||||
self.setheaders = SetHeaders()
|
||||
self.replay_ignore_params = False
|
||||
self.replay_ignore_params = False
|
||||
self.replay_ignore_content = None
|
||||
|
||||
|
||||
@ -515,11 +524,17 @@ class FlowMaster(controller.Master):
|
||||
for script in self.scripts:
|
||||
self.run_single_script_hook(script, name, *args, **kwargs)
|
||||
|
||||
def get_ignore(self):
|
||||
return [i.pattern for i in self.server.config.ignore]
|
||||
def get_ignore_filter(self):
|
||||
return self.server.config.check_ignore.patterns
|
||||
|
||||
def set_ignore(self, ignore):
|
||||
self.server.config.ignore = parse_host_pattern(ignore)
|
||||
def set_ignore_filter(self, host_patterns):
|
||||
self.server.config.check_ignore = HostMatcher(host_patterns)
|
||||
|
||||
def get_tcp_filter(self):
|
||||
return self.server.config.check_tcp.patterns
|
||||
|
||||
def set_tcp_filter(self, host_patterns):
|
||||
self.server.config.check_tcp = HostMatcher(host_patterns)
|
||||
|
||||
def set_stickycookie(self, txt):
|
||||
if txt:
|
||||
@ -601,7 +616,7 @@ class FlowMaster(controller.Master):
|
||||
]
|
||||
if all(e):
|
||||
self.shutdown()
|
||||
self.client_playback.tick(self, timeout)
|
||||
self.client_playback.tick(self)
|
||||
|
||||
return controller.Master.tick(self, q, timeout)
|
||||
|
||||
@ -612,6 +627,11 @@ class FlowMaster(controller.Master):
|
||||
"""
|
||||
Loads a flow, and returns a new flow object.
|
||||
"""
|
||||
|
||||
if self.server and self.server.config.mode == "reverse":
|
||||
f.request.host, f.request.port = self.server.config.mode.dst[2:]
|
||||
f.request.scheme = "https" if self.server.config.mode.dst[1] else "http"
|
||||
|
||||
f.reply = controller.DummyReply()
|
||||
if f.request:
|
||||
self.handle_request(f)
|
||||
@ -656,6 +676,8 @@ class FlowMaster(controller.Master):
|
||||
"""
|
||||
Returns None if successful, or error message if not.
|
||||
"""
|
||||
if f.live:
|
||||
return "Can't replay request which is still live..."
|
||||
if f.intercepting:
|
||||
return "Can't replay while intercepting..."
|
||||
if f.request.content == http.CONTENT_MISSING:
|
||||
@ -705,7 +727,11 @@ class FlowMaster(controller.Master):
|
||||
if f.live:
|
||||
app = self.apps.get(f.request)
|
||||
if app:
|
||||
err = app.serve(f, f.client_conn.wfile, **{"mitmproxy.master": self})
|
||||
err = app.serve(
|
||||
f,
|
||||
f.client_conn.wfile,
|
||||
**{"mitmproxy.master": self}
|
||||
)
|
||||
if err:
|
||||
self.add_event("Error in wsgi app. %s"%err, "error")
|
||||
f.reply(protocol.KILL)
|
||||
@ -720,8 +746,12 @@ class FlowMaster(controller.Master):
|
||||
def handle_responseheaders(self, f):
|
||||
self.run_script_hook("responseheaders", f)
|
||||
|
||||
if self.stream_large_bodies:
|
||||
self.stream_large_bodies.run(f, False)
|
||||
try:
|
||||
if self.stream_large_bodies:
|
||||
self.stream_large_bodies.run(f, False)
|
||||
except netlib.http.HttpError:
|
||||
f.reply(protocol.KILL)
|
||||
return
|
||||
|
||||
f.reply()
|
||||
return f
|
||||
@ -755,7 +785,6 @@ class FlowMaster(controller.Master):
|
||||
self.stream = None
|
||||
|
||||
|
||||
|
||||
class FlowWriter:
|
||||
def __init__(self, fo):
|
||||
self.fo = fo
|
||||
@ -787,7 +816,7 @@ class FlowReader:
|
||||
v = ".".join(str(i) for i in data["version"])
|
||||
raise FlowReadError("Incompatible serialized data version: %s"%v)
|
||||
off = self.fo.tell()
|
||||
yield handle.protocols[data["conntype"]]["flow"].from_state(data)
|
||||
yield handle.protocols[data["type"]]["flow"].from_state(data)
|
||||
except ValueError, v:
|
||||
# Error is due to EOF
|
||||
if self.fo.tell() == off and self.fo.read() == '':
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
from __future__ import print_function, absolute_import
|
||||
import argparse
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
@ -9,27 +8,43 @@ from .proxy import process_proxy_options, ProxyServerError
|
||||
from .proxy.server import DummyServer, ProxyServer
|
||||
|
||||
|
||||
# This file is not included in coverage analysis or tests - anything that can be
|
||||
# tested should live elsewhere.
|
||||
|
||||
def check_versions():
|
||||
"""
|
||||
Having installed a wrong version of pyOpenSSL or netlib is unfortunately a very common source of error.
|
||||
Check before every start that both versions are somewhat okay.
|
||||
Having installed a wrong version of pyOpenSSL or netlib is unfortunately a
|
||||
very common source of error. Check before every start that both versions are
|
||||
somewhat okay.
|
||||
"""
|
||||
# We don't introduce backward-incompatible changes in patch versions. Only consider major and minor version.
|
||||
# We don't introduce backward-incompatible changes in patch versions. Only
|
||||
# consider major and minor version.
|
||||
if netlib.version.IVERSION[:2] != version.IVERSION[:2]:
|
||||
print(
|
||||
"Warning: You are using mitmdump %s with netlib %s. "
|
||||
"Most likely, that doesn't work - please upgrade!" % (version.VERSION, netlib.version.VERSION),
|
||||
file=sys.stderr)
|
||||
import OpenSSL, inspect
|
||||
|
||||
"Most likely, that won't work - please upgrade!" % (
|
||||
version.VERSION, netlib.version.VERSION
|
||||
),
|
||||
file=sys.stderr
|
||||
)
|
||||
import OpenSSL
|
||||
import inspect
|
||||
v = tuple([int(x) for x in OpenSSL.__version__.split(".")][:2])
|
||||
if v < (0, 14):
|
||||
print("You are using an outdated version of pyOpenSSL: mitmproxy requires pyOpenSSL 0.14 or greater.",
|
||||
file=sys.stderr)
|
||||
# Some users apparently have multiple versions of pyOpenSSL installed. Report which one we got.
|
||||
print(
|
||||
"You are using an outdated version of pyOpenSSL:"
|
||||
" mitmproxy requires pyOpenSSL 0.14 or greater.",
|
||||
file=sys.stderr
|
||||
)
|
||||
# Some users apparently have multiple versions of pyOpenSSL installed.
|
||||
# Report which one we got.
|
||||
pyopenssl_path = os.path.dirname(inspect.getfile(OpenSSL))
|
||||
print("Your pyOpenSSL %s installation is located at %s" % (OpenSSL.__version__, pyopenssl_path),
|
||||
file=sys.stderr)
|
||||
print(
|
||||
"Your pyOpenSSL %s installation is located at %s" % (
|
||||
OpenSSL.__version__, pyopenssl_path
|
||||
),
|
||||
file=sys.stderr
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@ -38,8 +53,14 @@ def assert_utf8_env():
|
||||
for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
|
||||
spec += os.environ.get(i, "").lower()
|
||||
if "utf" not in spec:
|
||||
print("Error: mitmproxy requires a UTF console environment.", file=sys.stderr)
|
||||
print("Set your LANG enviroment variable to something like en_US.UTF-8", file=sys.stderr)
|
||||
print(
|
||||
"Error: mitmproxy requires a UTF console environment.",
|
||||
file=sys.stderr
|
||||
)
|
||||
print(
|
||||
"Set your LANG enviroment variable to something like en_US.UTF-8",
|
||||
file=sys.stderr
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@ -54,34 +75,13 @@ def get_server(dummy_server, options):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def mitmproxy_cmdline():
|
||||
# Don't import libmproxy.console for mitmdump, urwid is not available on all platforms.
|
||||
def mitmproxy(): # pragma: nocover
|
||||
from . import console
|
||||
from .console import palettes
|
||||
|
||||
parser = argparse.ArgumentParser(usage="%(prog)s [options]")
|
||||
parser.add_argument('--version', action='version', version=version.NAMEVERSION)
|
||||
cmdline.common_options(parser)
|
||||
parser.add_argument(
|
||||
"--palette", type=str, default="dark",
|
||||
action="store", dest="palette",
|
||||
help="Select color palette: " + ", ".join(palettes.palettes.keys())
|
||||
)
|
||||
parser.add_argument(
|
||||
"-e",
|
||||
action="store_true", dest="eventlog",
|
||||
help="Show event log."
|
||||
)
|
||||
group = parser.add_argument_group(
|
||||
"Filters",
|
||||
"See help in mitmproxy for filter expression syntax."
|
||||
)
|
||||
group.add_argument(
|
||||
"-i", "--intercept", action="store",
|
||||
type=str, dest="intercept", default=None,
|
||||
help="Intercept filter expression."
|
||||
)
|
||||
check_versions()
|
||||
assert_utf8_env()
|
||||
|
||||
parser = cmdline.mitmproxy()
|
||||
options = parser.parse_args()
|
||||
if options.quiet:
|
||||
options.verbose = 0
|
||||
@ -92,15 +92,6 @@ def mitmproxy_cmdline():
|
||||
console_options.eventlog = options.eventlog
|
||||
console_options.intercept = options.intercept
|
||||
|
||||
return console_options, proxy_config
|
||||
|
||||
|
||||
def mitmproxy(): # pragma: nocover
|
||||
from . import console
|
||||
|
||||
check_versions()
|
||||
assert_utf8_env()
|
||||
console_options, proxy_config = mitmproxy_cmdline()
|
||||
server = get_server(console_options.no_server, proxy_config)
|
||||
|
||||
m = console.ConsoleMaster(server, console_options)
|
||||
@ -110,24 +101,12 @@ def mitmproxy(): # pragma: nocover
|
||||
pass
|
||||
|
||||
|
||||
def mitmdump_cmdline():
|
||||
def mitmdump(): # pragma: nocover
|
||||
from . import dump
|
||||
|
||||
parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]")
|
||||
parser.add_argument('--version', action='version', version="mitmdump" + " " + version.VERSION)
|
||||
cmdline.common_options(parser)
|
||||
parser.add_argument(
|
||||
"--keepserving",
|
||||
action="store_true", dest="keepserving", default=False,
|
||||
help="Continue serving after client playback or file read. We exit by default."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
action="count", dest="flow_detail", default=1,
|
||||
help="Increase flow detail display level. Can be passed multiple times."
|
||||
)
|
||||
parser.add_argument('args', nargs=argparse.REMAINDER)
|
||||
check_versions()
|
||||
|
||||
parser = cmdline.mitmdump()
|
||||
options = parser.parse_args()
|
||||
if options.quiet:
|
||||
options.verbose = 0
|
||||
@ -139,14 +118,6 @@ def mitmdump_cmdline():
|
||||
dump_options.keepserving = options.keepserving
|
||||
dump_options.filtstr = " ".join(options.args) if options.args else None
|
||||
|
||||
return dump_options, proxy_config
|
||||
|
||||
|
||||
def mitmdump(): # pragma: nocover
|
||||
from . import dump
|
||||
|
||||
check_versions()
|
||||
dump_options, proxy_config = mitmdump_cmdline()
|
||||
server = get_server(dump_options.no_server, proxy_config)
|
||||
|
||||
try:
|
||||
@ -164,44 +135,11 @@ def mitmdump(): # pragma: nocover
|
||||
pass
|
||||
|
||||
|
||||
def mitmweb_cmdline():
|
||||
def mitmweb(): # pragma: nocover
|
||||
from . import web
|
||||
parser = argparse.ArgumentParser(usage="%(prog)s [options]")
|
||||
parser.add_argument(
|
||||
'--version',
|
||||
action='version',
|
||||
version="mitmweb" + " " + version.VERSION
|
||||
)
|
||||
|
||||
group = parser.add_argument_group("Mitmweb")
|
||||
group.add_argument(
|
||||
"--wport",
|
||||
action="store", type=int, dest="wport", default=8081,
|
||||
metavar="PORT",
|
||||
help="Mitmweb port."
|
||||
)
|
||||
group.add_argument(
|
||||
"--wiface",
|
||||
action="store", dest="wiface", default="127.0.0.1",
|
||||
metavar="IFACE",
|
||||
help="Mitmweb interface."
|
||||
)
|
||||
group.add_argument(
|
||||
"--wdebug",
|
||||
action="store_true", dest="wdebug",
|
||||
help="Turn on mitmweb debugging"
|
||||
)
|
||||
|
||||
cmdline.common_options(parser)
|
||||
group = parser.add_argument_group(
|
||||
"Filters",
|
||||
"See help in mitmproxy for filter expression syntax."
|
||||
)
|
||||
group.add_argument(
|
||||
"-i", "--intercept", action="store",
|
||||
type=str, dest="intercept", default=None,
|
||||
help="Intercept filter expression."
|
||||
)
|
||||
check_versions()
|
||||
parser = cmdline.mitmweb()
|
||||
|
||||
options = parser.parse_args()
|
||||
if options.quiet:
|
||||
@ -213,14 +151,7 @@ def mitmweb_cmdline():
|
||||
web_options.wdebug = options.wdebug
|
||||
web_options.wiface = options.wiface
|
||||
web_options.wport = options.wport
|
||||
return web_options, proxy_config
|
||||
|
||||
|
||||
def mitmweb(): # pragma: nocover
|
||||
from . import web
|
||||
|
||||
check_versions()
|
||||
web_options, proxy_config = mitmweb_cmdline()
|
||||
server = get_server(web_options.no_server, proxy_config)
|
||||
|
||||
m = web.WebMaster(server, web_options)
|
||||
|
||||
@ -18,12 +18,12 @@ def index():
|
||||
|
||||
@mapp.route("/cert/pem")
|
||||
def certs_pem():
|
||||
p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.pem")
|
||||
p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.pem")
|
||||
return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert')
|
||||
|
||||
|
||||
@mapp.route("/cert/p12")
|
||||
def certs_p12():
|
||||
p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.p12")
|
||||
p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.p12")
|
||||
return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12')
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
{% extends "frame.html" %}
|
||||
{% block body %}
|
||||
{% block body %}
|
||||
|
||||
<center>
|
||||
<h2> Click to install the mitmproxy certificate: </h2>
|
||||
@ -23,4 +23,13 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<hr/>
|
||||
<div class="text-center">
|
||||
Other mitmproxy users cannot intercept your connection.
|
||||
</div>
|
||||
<div class="text-center text-muted">
|
||||
This page is served by your local mitmproxy instance. The certificate you are about to install has been uniquely generated on mitmproxy's first run and is not shared
|
||||
between mitmproxy installations.
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import argparse
|
||||
import configargparse
|
||||
import cPickle as pickle
|
||||
from ctypes import byref, windll, Structure
|
||||
from ctypes.wintypes import DWORD
|
||||
@ -361,7 +361,7 @@ class TransparentProxy(object):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Windows Transparent Proxy")
|
||||
parser = configargparse.ArgumentParser(description="Windows Transparent Proxy")
|
||||
parser.add_argument('--mode', choices=['forward', 'local', 'both'], default="both",
|
||||
help='redirection operation mode: "forward" to only redirect forwarded packets, '
|
||||
'"local" to only redirect packets originating from the local machine')
|
||||
|
||||
@ -18,12 +18,17 @@ HDR_FORM_URLENCODED = "application/x-www-form-urlencoded"
|
||||
CONTENT_MISSING = 0
|
||||
|
||||
|
||||
class KillSignal(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def get_line(fp):
|
||||
"""
|
||||
Get a line, possibly preceded by a blank.
|
||||
"""
|
||||
line = fp.readline()
|
||||
if line == "\r\n" or line == "\n": # Possible leftover from previous message
|
||||
if line == "\r\n" or line == "\n":
|
||||
# Possible leftover from previous message
|
||||
line = fp.readline()
|
||||
if line == "":
|
||||
raise tcp.NetLibDisconnect()
|
||||
@ -237,25 +242,47 @@ class HTTPRequest(HTTPMessage):
|
||||
is content associated, but not present. CONTENT_MISSING evaluates
|
||||
to False to make checking for the presence of content natural.
|
||||
|
||||
form_in: The request form which mitmproxy has received. The following values are possible:
|
||||
- relative (GET /index.html, OPTIONS *) (covers origin form and asterisk form)
|
||||
- absolute (GET http://example.com:80/index.html)
|
||||
- authority-form (CONNECT example.com:443)
|
||||
Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3
|
||||
form_in: The request form which mitmproxy has received. The following
|
||||
values are possible:
|
||||
|
||||
form_out: The request form which mitmproxy has send out to the destination
|
||||
- relative (GET /index.html, OPTIONS *) (covers origin form and
|
||||
asterisk form)
|
||||
- absolute (GET http://example.com:80/index.html)
|
||||
- authority-form (CONNECT example.com:443)
|
||||
Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3
|
||||
|
||||
form_out: The request form which mitmproxy will send out to the
|
||||
destination
|
||||
|
||||
timestamp_start: Timestamp indicating when request transmission started
|
||||
|
||||
timestamp_end: Timestamp indicating when request transmission ended
|
||||
"""
|
||||
|
||||
def __init__(self, form_in, method, scheme, host, port, path, httpversion, headers,
|
||||
content, timestamp_start=None, timestamp_end=None, form_out=None):
|
||||
def __init__(
|
||||
self,
|
||||
form_in,
|
||||
method,
|
||||
scheme,
|
||||
host,
|
||||
port,
|
||||
path,
|
||||
httpversion,
|
||||
headers,
|
||||
content,
|
||||
timestamp_start=None,
|
||||
timestamp_end=None,
|
||||
form_out=None
|
||||
):
|
||||
assert isinstance(headers, ODictCaseless) or not headers
|
||||
HTTPMessage.__init__(self, httpversion, headers, content, timestamp_start,
|
||||
timestamp_end)
|
||||
|
||||
HTTPMessage.__init__(
|
||||
self,
|
||||
httpversion,
|
||||
headers,
|
||||
content,
|
||||
timestamp_start,
|
||||
timestamp_end
|
||||
)
|
||||
self.form_in = form_in
|
||||
self.method = method
|
||||
self.scheme = scheme
|
||||
@ -308,30 +335,43 @@ class HTTPRequest(HTTPMessage):
|
||||
|
||||
request_line = get_line(rfile)
|
||||
|
||||
if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start
|
||||
if hasattr(rfile, "first_byte_timestamp"):
|
||||
# more accurate timestamp_start
|
||||
timestamp_start = rfile.first_byte_timestamp
|
||||
|
||||
request_line_parts = http.parse_init(request_line)
|
||||
if not request_line_parts:
|
||||
raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
|
||||
raise http.HttpError(
|
||||
400,
|
||||
"Bad HTTP request line: %s" % repr(request_line)
|
||||
)
|
||||
method, path, httpversion = request_line_parts
|
||||
|
||||
if path == '*' or path.startswith("/"):
|
||||
form_in = "relative"
|
||||
if not netlib.utils.isascii(path):
|
||||
raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
|
||||
raise http.HttpError(
|
||||
400,
|
||||
"Bad HTTP request line: %s" % repr(request_line)
|
||||
)
|
||||
elif method.upper() == 'CONNECT':
|
||||
form_in = "authority"
|
||||
r = http.parse_init_connect(request_line)
|
||||
if not r:
|
||||
raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
|
||||
raise http.HttpError(
|
||||
400,
|
||||
"Bad HTTP request line: %s" % repr(request_line)
|
||||
)
|
||||
host, port, _ = r
|
||||
path = None
|
||||
else:
|
||||
form_in = "absolute"
|
||||
r = http.parse_init_proxy(request_line)
|
||||
if not r:
|
||||
raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
|
||||
raise http.HttpError(
|
||||
400,
|
||||
"Bad HTTP request line: %s" % repr(request_line)
|
||||
)
|
||||
_, scheme, host, port, path, _ = r
|
||||
|
||||
headers = http.read_headers(rfile)
|
||||
@ -343,50 +383,69 @@ class HTTPRequest(HTTPMessage):
|
||||
method, None, True)
|
||||
timestamp_end = utils.timestamp()
|
||||
|
||||
return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers,
|
||||
content, timestamp_start, timestamp_end)
|
||||
return HTTPRequest(
|
||||
form_in,
|
||||
method,
|
||||
scheme,
|
||||
host,
|
||||
port,
|
||||
path,
|
||||
httpversion,
|
||||
headers,
|
||||
content,
|
||||
timestamp_start,
|
||||
timestamp_end
|
||||
)
|
||||
|
||||
def _assemble_first_line(self, form=None):
|
||||
form = form or self.form_out
|
||||
|
||||
if form == "relative":
|
||||
path = self.path if self.method != "OPTIONS" else "*"
|
||||
request_line = '%s %s HTTP/%s.%s' % \
|
||||
(self.method, path, self.httpversion[0], self.httpversion[1])
|
||||
request_line = '%s %s HTTP/%s.%s' % (
|
||||
self.method, self.path, self.httpversion[0], self.httpversion[1]
|
||||
)
|
||||
elif form == "authority":
|
||||
request_line = '%s %s:%s HTTP/%s.%s' % (self.method, self.host, self.port,
|
||||
self.httpversion[0], self.httpversion[1])
|
||||
request_line = '%s %s:%s HTTP/%s.%s' % (
|
||||
self.method, self.host, self.port, self.httpversion[0],
|
||||
self.httpversion[1]
|
||||
)
|
||||
elif form == "absolute":
|
||||
request_line = '%s %s://%s:%s%s HTTP/%s.%s' % \
|
||||
(self.method, self.scheme, self.host, self.port, self.path,
|
||||
self.httpversion[0], self.httpversion[1])
|
||||
request_line = '%s %s://%s:%s%s HTTP/%s.%s' % (
|
||||
self.method, self.scheme, self.host,
|
||||
self.port, self.path, self.httpversion[0],
|
||||
self.httpversion[1]
|
||||
)
|
||||
else:
|
||||
raise http.HttpError(400, "Invalid request form")
|
||||
return request_line
|
||||
|
||||
# This list is adopted legacy code.
|
||||
# We probably don't need to strip off keep-alive.
|
||||
_headers_to_strip_off = ['Proxy-Connection',
|
||||
'Keep-Alive',
|
||||
'Connection',
|
||||
'Transfer-Encoding',
|
||||
'Upgrade']
|
||||
|
||||
def _assemble_headers(self):
|
||||
headers = self.headers.copy()
|
||||
for k in ['Proxy-Connection',
|
||||
'Keep-Alive',
|
||||
'Connection',
|
||||
'Transfer-Encoding']:
|
||||
for k in self._headers_to_strip_off:
|
||||
del headers[k]
|
||||
if headers["Upgrade"] == ["h2c"]: # Suppress HTTP2 https://http2.github.io/http2-spec/index.html#discover-http
|
||||
del headers["Upgrade"]
|
||||
if not 'host' in headers and self.scheme and self.host and self.port:
|
||||
if 'host' not in headers and self.scheme and self.host and self.port:
|
||||
headers["Host"] = [utils.hostport(self.scheme,
|
||||
self.host,
|
||||
self.port)]
|
||||
|
||||
if self.content:
|
||||
# If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header.
|
||||
if self.content or self.content == "":
|
||||
headers["Content-Length"] = [str(len(self.content))]
|
||||
elif 'Transfer-Encoding' in self.headers: # content-length for e.g. chuncked transfer-encoding with no content
|
||||
headers["Content-Length"] = ["0"]
|
||||
|
||||
return str(headers)
|
||||
|
||||
def _assemble_head(self, form=None):
|
||||
return "%s\r\n%s\r\n" % (self._assemble_first_line(form), self._assemble_headers())
|
||||
return "%s\r\n%s\r\n" % (
|
||||
self._assemble_first_line(form), self._assemble_headers()
|
||||
)
|
||||
|
||||
def assemble(self, form=None):
|
||||
"""
|
||||
@ -396,7 +455,10 @@ class HTTPRequest(HTTPMessage):
|
||||
Raises an Exception if the request cannot be assembled.
|
||||
"""
|
||||
if self.content == CONTENT_MISSING:
|
||||
raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING")
|
||||
raise proxy.ProxyError(
|
||||
502,
|
||||
"Cannot assemble flow with CONTENT_MISSING"
|
||||
)
|
||||
head = self._assemble_head(form)
|
||||
if self.content:
|
||||
return head + self.content
|
||||
@ -644,7 +706,9 @@ class HTTPResponse(HTTPMessage):
|
||||
return "<HTTPResponse: {code} {msg} ({contenttype}, {size})>".format(
|
||||
code=self.code,
|
||||
msg=self.msg,
|
||||
contenttype=self.headers.get_first("content-type", "unknown content type"),
|
||||
contenttype=self.headers.get_first(
|
||||
"content-type", "unknown content type"
|
||||
),
|
||||
size=size
|
||||
)
|
||||
|
||||
@ -665,7 +729,8 @@ class HTTPResponse(HTTPMessage):
|
||||
body_size_limit,
|
||||
include_body=include_body)
|
||||
|
||||
if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start
|
||||
if hasattr(rfile, "first_byte_timestamp"):
|
||||
# more accurate timestamp_start
|
||||
timestamp_start = rfile.first_byte_timestamp
|
||||
|
||||
if include_body:
|
||||
@ -687,26 +752,30 @@ class HTTPResponse(HTTPMessage):
|
||||
return 'HTTP/%s.%s %s %s' % \
|
||||
(self.httpversion[0], self.httpversion[1], self.code, self.msg)
|
||||
|
||||
_headers_to_strip_off = ['Proxy-Connection',
|
||||
'Alternate-Protocol',
|
||||
'Alt-Svc']
|
||||
|
||||
def _assemble_headers(self, preserve_transfer_encoding=False):
|
||||
headers = self.headers.copy()
|
||||
for k in ['Proxy-Connection',
|
||||
'Alternate-Protocol',
|
||||
'Alt-Svc']:
|
||||
for k in self._headers_to_strip_off:
|
||||
del headers[k]
|
||||
if not preserve_transfer_encoding:
|
||||
del headers['Transfer-Encoding']
|
||||
|
||||
if self.content:
|
||||
# If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header.
|
||||
if self.content or self.content == "":
|
||||
headers["Content-Length"] = [str(len(self.content))]
|
||||
# add content-length for chuncked transfer-encoding with no content
|
||||
elif not preserve_transfer_encoding and 'Transfer-Encoding' in self.headers:
|
||||
headers["Content-Length"] = ["0"]
|
||||
|
||||
return str(headers)
|
||||
|
||||
def _assemble_head(self, preserve_transfer_encoding=False):
|
||||
return '%s\r\n%s\r\n' % (
|
||||
self._assemble_first_line(), self._assemble_headers(preserve_transfer_encoding=preserve_transfer_encoding))
|
||||
self._assemble_first_line(),
|
||||
self._assemble_headers(
|
||||
preserve_transfer_encoding=preserve_transfer_encoding
|
||||
)
|
||||
)
|
||||
|
||||
def assemble(self):
|
||||
"""
|
||||
@ -716,7 +785,10 @@ class HTTPResponse(HTTPMessage):
|
||||
Raises an Exception if the request cannot be assembled.
|
||||
"""
|
||||
if self.content == CONTENT_MISSING:
|
||||
raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING")
|
||||
raise proxy.ProxyError(
|
||||
502,
|
||||
"Cannot assemble flow with CONTENT_MISSING"
|
||||
)
|
||||
head = self._assemble_head()
|
||||
if self.content:
|
||||
return head + self.content
|
||||
@ -783,8 +855,9 @@ class HTTPResponse(HTTPMessage):
|
||||
pairs = [pair.partition("=") for pair in header.split(';')]
|
||||
cookie_name = pairs[0][0] # the key of the first key/value pairs
|
||||
cookie_value = pairs[0][2] # the value of the first key/value pairs
|
||||
cookie_parameters = {key.strip().lower(): value.strip() for key, sep, value in
|
||||
pairs[1:]}
|
||||
cookie_parameters = {
|
||||
key.strip().lower(): value.strip() for key, sep, value in pairs[1:]
|
||||
}
|
||||
cookies.append((cookie_name, (cookie_value, cookie_parameters)))
|
||||
return dict(cookies)
|
||||
|
||||
@ -817,7 +890,8 @@ class HTTPFlow(Flow):
|
||||
self.response = None
|
||||
"""@type: HTTPResponse"""
|
||||
|
||||
self.intercepting = False # FIXME: Should that rather be an attribute of Flow?
|
||||
# FIXME: Should that rather be an attribute of Flow?
|
||||
self.intercepting = False
|
||||
|
||||
_stateobject_attributes = Flow._stateobject_attributes.copy()
|
||||
_stateobject_attributes.update(
|
||||
@ -905,7 +979,9 @@ class HTTPFlow(Flow):
|
||||
|
||||
class HttpAuthenticationError(Exception):
|
||||
def __init__(self, auth_headers=None):
|
||||
super(HttpAuthenticationError, self).__init__("Proxy Authentication Required")
|
||||
super(HttpAuthenticationError, self).__init__(
|
||||
"Proxy Authentication Required"
|
||||
)
|
||||
self.headers = auth_headers
|
||||
self.code = 407
|
||||
|
||||
@ -937,16 +1013,23 @@ class HTTPHandler(ProtocolHandler):
|
||||
try:
|
||||
self.c.server_conn.send(request_raw)
|
||||
# Only get the headers at first...
|
||||
flow.response = HTTPResponse.from_stream(self.c.server_conn.rfile, flow.request.method,
|
||||
body_size_limit=self.c.config.body_size_limit,
|
||||
include_body=False)
|
||||
flow.response = HTTPResponse.from_stream(
|
||||
self.c.server_conn.rfile, flow.request.method,
|
||||
body_size_limit=self.c.config.body_size_limit,
|
||||
include_body=False
|
||||
)
|
||||
break
|
||||
except (tcp.NetLibDisconnect, http.HttpErrorConnClosed), v:
|
||||
self.c.log("error in server communication: %s" % repr(v), level="debug")
|
||||
self.c.log(
|
||||
"error in server communication: %s" % repr(v),
|
||||
level="debug"
|
||||
)
|
||||
if attempt == 0:
|
||||
# In any case, we try to reconnect at least once.
|
||||
# This is necessary because it might be possible that we already initiated an upstream connection
|
||||
# after clientconnect that has already been expired, e.g consider the following event log:
|
||||
# In any case, we try to reconnect at least once. This is
|
||||
# necessary because it might be possible that we already
|
||||
# initiated an upstream connection after clientconnect that
|
||||
# has already been expired, e.g consider the following event
|
||||
# log:
|
||||
# > clientconnect (transparent mode destination known)
|
||||
# > serverconnect
|
||||
# > read n% of large request
|
||||
@ -959,19 +1042,21 @@ class HTTPHandler(ProtocolHandler):
|
||||
|
||||
# call the appropriate script hook - this is an opportunity for an
|
||||
# inline script to set flow.stream = True
|
||||
self.c.channel.ask("responseheaders", flow)
|
||||
|
||||
# now get the rest of the request body, if body still needs to be read
|
||||
# but not streaming this response
|
||||
if flow.response.stream:
|
||||
flow.response.content = CONTENT_MISSING
|
||||
flow = self.c.channel.ask("responseheaders", flow)
|
||||
if flow is None or flow == KILL:
|
||||
raise KillSignal()
|
||||
else:
|
||||
flow.response.content = http.read_http_body(
|
||||
self.c.server_conn.rfile, flow.response.headers,
|
||||
self.c.config.body_size_limit,
|
||||
flow.request.method, flow.response.code, False
|
||||
)
|
||||
flow.response.timestamp_end = utils.timestamp()
|
||||
# now get the rest of the request body, if body still needs to be
|
||||
# read but not streaming this response
|
||||
if flow.response.stream:
|
||||
flow.response.content = CONTENT_MISSING
|
||||
else:
|
||||
flow.response.content = http.read_http_body(
|
||||
self.c.server_conn.rfile, flow.response.headers,
|
||||
self.c.config.body_size_limit,
|
||||
flow.request.method, flow.response.code, False
|
||||
)
|
||||
flow.response.timestamp_end = utils.timestamp()
|
||||
|
||||
def handle_flow(self):
|
||||
flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live)
|
||||
@ -1001,10 +1086,10 @@ class HTTPHandler(ProtocolHandler):
|
||||
# sent through to the Master.
|
||||
flow.request = req
|
||||
request_reply = self.c.channel.ask("request", flow)
|
||||
self.process_server_address(flow) # The inline script may have changed request.host
|
||||
|
||||
if request_reply is None or request_reply == KILL:
|
||||
return False
|
||||
raise KillSignal()
|
||||
|
||||
self.process_server_address(flow) # The inline script may have changed request.host
|
||||
|
||||
if isinstance(request_reply, HTTPResponse):
|
||||
flow.response = request_reply
|
||||
@ -1018,7 +1103,7 @@ class HTTPHandler(ProtocolHandler):
|
||||
self.c.log("response", "debug", [flow.response._assemble_first_line()])
|
||||
response_reply = self.c.channel.ask("response", flow)
|
||||
if response_reply is None or response_reply == KILL:
|
||||
return False
|
||||
raise KillSignal()
|
||||
|
||||
self.send_response_to_client(flow)
|
||||
|
||||
@ -1050,15 +1135,27 @@ class HTTPHandler(ProtocolHandler):
|
||||
flow.live.restore_server()
|
||||
|
||||
return True # Next flow please.
|
||||
except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e:
|
||||
except (
|
||||
HttpAuthenticationError,
|
||||
http.HttpError,
|
||||
proxy.ProxyError,
|
||||
tcp.NetLibError,
|
||||
), e:
|
||||
self.handle_error(e, flow)
|
||||
except KillSignal:
|
||||
self.c.log("Connection killed", "info")
|
||||
finally:
|
||||
flow.live = None # Connection is not live anymore.
|
||||
return False
|
||||
|
||||
def handle_server_reconnect(self, state):
|
||||
if state["state"] == "connect":
|
||||
send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False)
|
||||
send_connect_request(
|
||||
self.c.server_conn,
|
||||
state["host"],
|
||||
state["port"],
|
||||
update_state=False
|
||||
)
|
||||
else: # pragma: nocover
|
||||
raise RuntimeError("Unknown State: %s" % state["state"])
|
||||
|
||||
@ -1079,14 +1176,14 @@ class HTTPHandler(ProtocolHandler):
|
||||
if message:
|
||||
self.c.log(message, level="info")
|
||||
if message_debug:
|
||||
self.c.log(message, level="debug")
|
||||
self.c.log(message_debug, level="debug")
|
||||
|
||||
if flow:
|
||||
# TODO: no flows without request or with both request and response at the moment.
|
||||
# TODO: no flows without request or with both request and response
|
||||
# at the moment.
|
||||
if flow.request and not flow.response:
|
||||
flow.error = Error(message or message_debug)
|
||||
self.c.channel.ask("error", flow)
|
||||
|
||||
try:
|
||||
code = getattr(error, "code", 502)
|
||||
headers = getattr(error, "headers", None)
|
||||
@ -1100,12 +1197,22 @@ class HTTPHandler(ProtocolHandler):
|
||||
|
||||
def send_error(self, code, message, headers):
|
||||
response = http_status.RESPONSES.get(code, "Unknown")
|
||||
html_content = '<html><head>\n<title>%d %s</title>\n</head>\n<body>\n%s\n</body>\n</html>' % \
|
||||
(code, response, message)
|
||||
html_content = """
|
||||
<html>
|
||||
<head>
|
||||
<title>%d %s</title>
|
||||
</head>
|
||||
<body>%s</body>
|
||||
</html>
|
||||
""" % (code, response, message)
|
||||
self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response))
|
||||
self.c.client_conn.wfile.write("Server: %s\r\n" % self.c.config.server_version)
|
||||
self.c.client_conn.wfile.write(
|
||||
"Server: %s\r\n" % self.c.config.server_version
|
||||
)
|
||||
self.c.client_conn.wfile.write("Content-type: text/html\r\n")
|
||||
self.c.client_conn.wfile.write("Content-Length: %d\r\n" % len(html_content))
|
||||
self.c.client_conn.wfile.write(
|
||||
"Content-Length: %d\r\n" % len(html_content)
|
||||
)
|
||||
if headers:
|
||||
for key, value in headers.items():
|
||||
self.c.client_conn.wfile.write("%s: %s\r\n" % (key, value))
|
||||
@ -1145,11 +1252,15 @@ class HTTPHandler(ProtocolHandler):
|
||||
# Now we can process the request.
|
||||
if request.form_in == "authority":
|
||||
if self.c.client_conn.ssl_established:
|
||||
raise http.HttpError(400, "Must not CONNECT on already encrypted connection")
|
||||
raise http.HttpError(
|
||||
400,
|
||||
"Must not CONNECT on already encrypted connection"
|
||||
)
|
||||
|
||||
if self.c.config.mode == "regular":
|
||||
self.c.set_server_address((request.host, request.port))
|
||||
flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow
|
||||
# Update server_conn attribute on the flow
|
||||
flow.server_conn = self.c.server_conn
|
||||
self.c.establish_server_connection()
|
||||
self.c.client_conn.send(
|
||||
'HTTP/1.1 200 Connection established\r\n' +
|
||||
@ -1161,7 +1272,9 @@ class HTTPHandler(ProtocolHandler):
|
||||
elif self.c.config.mode == "upstream":
|
||||
return None
|
||||
else:
|
||||
pass # CONNECT should never occur if we don't expect absolute-form requests
|
||||
# CONNECT should never occur if we don't expect absolute-form
|
||||
# requests
|
||||
pass
|
||||
|
||||
elif request.form_in == self.expected_form_in:
|
||||
|
||||
@ -1169,61 +1282,77 @@ class HTTPHandler(ProtocolHandler):
|
||||
|
||||
if request.form_in == "absolute":
|
||||
if request.scheme != "http":
|
||||
raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme)
|
||||
raise http.HttpError(
|
||||
400,
|
||||
"Invalid request scheme: %s" % request.scheme
|
||||
)
|
||||
if self.c.config.mode == "regular":
|
||||
# Update info so that an inline script sees the correct value at flow.server_conn
|
||||
# Update info so that an inline script sees the correct
|
||||
# value at flow.server_conn
|
||||
self.c.set_server_address((request.host, request.port))
|
||||
flow.server_conn = self.c.server_conn
|
||||
|
||||
return None
|
||||
|
||||
raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" %
|
||||
(self.expected_form_in, request.form_in))
|
||||
raise http.HttpError(
|
||||
400, "Invalid HTTP request form (expected: %s, got: %s)" % (
|
||||
self.expected_form_in, request.form_in
|
||||
)
|
||||
)
|
||||
|
||||
def process_server_address(self, flow):
|
||||
# Depending on the proxy mode, server handling is entirely different
|
||||
# We provide a mostly unified API to the user, which needs to be unfiddled here
|
||||
# We provide a mostly unified API to the user, which needs to be
|
||||
# unfiddled here
|
||||
# ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 )
|
||||
address = netlib.tcp.Address((flow.request.host, flow.request.port))
|
||||
|
||||
ssl = (flow.request.scheme == "https")
|
||||
|
||||
if self.c.config.mode == "upstream":
|
||||
|
||||
# The connection to the upstream proxy may have a state we may need to take into account.
|
||||
# The connection to the upstream proxy may have a state we may need
|
||||
# to take into account.
|
||||
connected_to = None
|
||||
for s in flow.server_conn.state:
|
||||
if s[0] == "http" and s[1]["state"] == "connect":
|
||||
connected_to = tcp.Address((s[1]["host"], s[1]["port"]))
|
||||
|
||||
# We need to reconnect if the current flow either requires a (possibly impossible)
|
||||
# change to the connection state, e.g. the host has changed but we already CONNECTed somewhere else.
|
||||
# We need to reconnect if the current flow either requires a
|
||||
# (possibly impossible) change to the connection state, e.g. the
|
||||
# host has changed but we already CONNECTed somewhere else.
|
||||
needs_server_change = (
|
||||
ssl != self.c.server_conn.ssl_established
|
||||
or
|
||||
(connected_to and address != connected_to) # HTTP proxying is "stateless", CONNECT isn't.
|
||||
# HTTP proxying is "stateless", CONNECT isn't.
|
||||
(connected_to and address != connected_to)
|
||||
)
|
||||
|
||||
if needs_server_change:
|
||||
# force create new connection to the proxy server to reset state
|
||||
self.live.change_server(self.c.server_conn.address, force=True)
|
||||
if ssl:
|
||||
send_connect_request(self.c.server_conn, address.host, address.port)
|
||||
send_connect_request(
|
||||
self.c.server_conn,
|
||||
address.host,
|
||||
address.port
|
||||
)
|
||||
self.c.establish_ssl(server=True)
|
||||
else:
|
||||
# If we're not in upstream mode, we just want to update the host and possibly establish TLS.
|
||||
self.live.change_server(address, ssl=ssl) # this is a no op if the addresses match.
|
||||
# If we're not in upstream mode, we just want to update the host and
|
||||
# possibly establish TLS. This is a no op if the addresses match.
|
||||
self.live.change_server(address, ssl=ssl)
|
||||
|
||||
flow.server_conn = self.c.server_conn
|
||||
|
||||
def send_response_to_client(self, flow):
|
||||
if not flow.response.stream:
|
||||
# no streaming:
|
||||
# we already received the full response from the server and can send it to the client straight away.
|
||||
# we already received the full response from the server and can send
|
||||
# it to the client straight away.
|
||||
self.c.client_conn.send(flow.response.assemble())
|
||||
else:
|
||||
# streaming:
|
||||
# First send the headers and then transfer the response incrementally:
|
||||
# First send the headers and then transfer the response
|
||||
# incrementally:
|
||||
h = flow.response._assemble_head(preserve_transfer_encoding=True)
|
||||
self.c.client_conn.send(h)
|
||||
for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
|
||||
@ -1237,7 +1366,8 @@ class HTTPHandler(ProtocolHandler):
|
||||
|
||||
def check_close_connection(self, flow):
|
||||
"""
|
||||
Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so.
|
||||
Checks if the connection should be closed depending on the HTTP
|
||||
semantics. Returns True, if so.
|
||||
"""
|
||||
close_connection = (
|
||||
http.connection_close(flow.request.httpversion, flow.request.headers) or
|
||||
@ -1260,20 +1390,39 @@ class HTTPHandler(ProtocolHandler):
|
||||
Returns False, if the connection should be closed immediately.
|
||||
"""
|
||||
address = tcp.Address.wrap(address)
|
||||
if self.c.check_ignore_address(address):
|
||||
if self.c.config.check_ignore(address):
|
||||
self.c.log("Ignore host: %s:%s" % address(), "info")
|
||||
TCPHandler(self.c).handle_messages()
|
||||
TCPHandler(self.c, log=False).handle_messages()
|
||||
return False
|
||||
else:
|
||||
self.expected_form_in = "relative"
|
||||
self.expected_form_out = "relative"
|
||||
self.skip_authentication = True
|
||||
|
||||
if address.port in self.c.config.ssl_ports:
|
||||
# In practice, nobody issues a CONNECT request to send unencrypted HTTP requests afterwards.
|
||||
# If we don't delegate to TCP mode, we should always negotiate a SSL connection.
|
||||
#
|
||||
# FIXME:
|
||||
# Turns out the previous statement isn't entirely true. Chrome on Windows CONNECTs to :80
|
||||
# if an explicit proxy is configured and a websocket connection should be established.
|
||||
# We don't support websocket at the moment, so it fails anyway, but we should come up with
|
||||
# a better solution to this if we start to support WebSockets.
|
||||
should_establish_ssl = (
|
||||
address.port in self.c.config.ssl_ports
|
||||
or
|
||||
not self.c.config.check_tcp(address)
|
||||
)
|
||||
|
||||
if should_establish_ssl:
|
||||
self.c.log("Received CONNECT request to SSL port. Upgrading to SSL...", "debug")
|
||||
self.c.establish_ssl(server=True, client=True)
|
||||
self.c.log("Upgrade to SSL completed.", "debug")
|
||||
|
||||
if self.c.config.check_tcp(address):
|
||||
self.c.log("Generic TCP mode for host: %s:%s" % address(), "info")
|
||||
TCPHandler(self.c).handle_messages()
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def authenticate(self, request):
|
||||
@ -1297,31 +1446,43 @@ class RequestReplayThread(threading.Thread):
|
||||
r = self.flow.request
|
||||
form_out_backup = r.form_out
|
||||
try:
|
||||
# In all modes, we directly connect to the server displayed
|
||||
if self.config.mode == "upstream":
|
||||
server_address = self.config.mode.get_upstream_server(self.flow.client_conn)[2:]
|
||||
server = ServerConnection(server_address)
|
||||
server.connect()
|
||||
if r.scheme == "https":
|
||||
send_connect_request(server, r.host, r.port)
|
||||
server.establish_ssl(self.config.clientcerts, sni=r.host)
|
||||
r.form_out = "relative"
|
||||
else:
|
||||
r.form_out = "absolute"
|
||||
self.flow.response = None
|
||||
request_reply = self.channel.ask("request", self.flow)
|
||||
if request_reply is None or request_reply == KILL:
|
||||
raise KillSignal()
|
||||
elif isinstance(request_reply, HTTPResponse):
|
||||
self.flow.response = request_reply
|
||||
else:
|
||||
server_address = (r.host, r.port)
|
||||
server = ServerConnection(server_address)
|
||||
server.connect()
|
||||
if r.scheme == "https":
|
||||
server.establish_ssl(self.config.clientcerts, sni=r.host)
|
||||
r.form_out = "relative"
|
||||
# In all modes, we directly connect to the server displayed
|
||||
if self.config.mode == "upstream":
|
||||
server_address = self.config.mode.get_upstream_server(self.flow.client_conn)[2:]
|
||||
server = ServerConnection(server_address)
|
||||
server.connect()
|
||||
if r.scheme == "https":
|
||||
send_connect_request(server, r.host, r.port)
|
||||
server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni)
|
||||
r.form_out = "relative"
|
||||
else:
|
||||
r.form_out = "absolute"
|
||||
else:
|
||||
server_address = (r.host, r.port)
|
||||
server = ServerConnection(server_address)
|
||||
server.connect()
|
||||
if r.scheme == "https":
|
||||
server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni)
|
||||
r.form_out = "relative"
|
||||
|
||||
server.send(r.assemble())
|
||||
self.flow.response = HTTPResponse.from_stream(server.rfile, r.method,
|
||||
body_size_limit=self.config.body_size_limit)
|
||||
self.channel.ask("response", self.flow)
|
||||
except (proxy.ProxyError, http.HttpError, tcp.NetLibError), v:
|
||||
server.send(r.assemble())
|
||||
self.flow.server_conn = server
|
||||
self.flow.response = HTTPResponse.from_stream(server.rfile, r.method,
|
||||
body_size_limit=self.config.body_size_limit)
|
||||
response_reply = self.channel.ask("response", self.flow)
|
||||
if response_reply is None or response_reply == KILL:
|
||||
raise KillSignal()
|
||||
except (proxy.ProxyError, http.HttpError, tcp.NetLibError) as v:
|
||||
self.flow.error = Error(repr(v))
|
||||
self.channel.ask("error", self.flow)
|
||||
except KillSignal:
|
||||
self.channel.tell("log", proxy.Log("Connection killed", "info"))
|
||||
finally:
|
||||
r.form_out = form_out_backup
|
||||
|
||||
@ -59,8 +59,8 @@ class Flow(stateobject.StateObject):
|
||||
A Flow is a collection of objects representing a single transaction.
|
||||
This class is usually subclassed for each protocol, e.g. HTTPFlow.
|
||||
"""
|
||||
def __init__(self, conntype, client_conn, server_conn, live=None):
|
||||
self.conntype = conntype
|
||||
def __init__(self, type, client_conn, server_conn, live=None):
|
||||
self.type = type
|
||||
self.id = str(uuid.uuid4())
|
||||
self.client_conn = client_conn
|
||||
"""@type: ClientConnection"""
|
||||
@ -78,7 +78,7 @@ class Flow(stateobject.StateObject):
|
||||
error=Error,
|
||||
client_conn=ClientConnection,
|
||||
server_conn=ServerConnection,
|
||||
conntype=str
|
||||
type=str
|
||||
)
|
||||
|
||||
def get_state(self, short=False):
|
||||
@ -174,7 +174,7 @@ class LiveConnection(object):
|
||||
self._backup_server_conn = None
|
||||
"""@type: libmproxy.proxy.connection.ServerConnection"""
|
||||
|
||||
def change_server(self, address, ssl=None, force=False, persistent_change=False):
|
||||
def change_server(self, address, ssl=None, sni=None, force=False, persistent_change=False):
|
||||
"""
|
||||
Change the server connection to the specified address.
|
||||
@returns:
|
||||
@ -183,7 +183,14 @@ class LiveConnection(object):
|
||||
"""
|
||||
address = netlib.tcp.Address.wrap(address)
|
||||
|
||||
ssl_mismatch = (ssl is not None and ssl != self.c.server_conn.ssl_established)
|
||||
ssl_mismatch = (
|
||||
ssl is not None and
|
||||
(
|
||||
ssl != self.c.server_conn.ssl_established
|
||||
or
|
||||
(sni is not None and sni != self.c.sni)
|
||||
)
|
||||
)
|
||||
address_mismatch = (address != self.c.server_conn.address)
|
||||
|
||||
if persistent_change:
|
||||
@ -212,6 +219,8 @@ class LiveConnection(object):
|
||||
|
||||
self.c.set_server_address(address)
|
||||
self.c.establish_server_connection(ask=False)
|
||||
if sni:
|
||||
self.c.sni = sni
|
||||
if ssl:
|
||||
self.c.establish_ssl(server=True)
|
||||
return True
|
||||
|
||||
@ -13,6 +13,10 @@ class TCPHandler(ProtocolHandler):
|
||||
|
||||
chunk_size = 4096
|
||||
|
||||
def __init__(self, c, log=True):
|
||||
super(TCPHandler, self).__init__(c)
|
||||
self.log = log
|
||||
|
||||
def handle_messages(self):
|
||||
self.c.establish_server_connection()
|
||||
|
||||
@ -63,26 +67,25 @@ class TCPHandler(ProtocolHandler):
|
||||
# if one of the peers is over SSL, we need to send
|
||||
# bytes/strings
|
||||
if not src.ssl_established:
|
||||
# only ssl to dst, i.e. we revc'd into buf but need
|
||||
# bytes/string now.
|
||||
# we revc'd into buf but need bytes/string now.
|
||||
contents = buf[:size].tobytes()
|
||||
self.c.log(
|
||||
"%s %s\r\n%s" % (
|
||||
direction, dst_str, cleanBin(contents)
|
||||
),
|
||||
"debug"
|
||||
)
|
||||
if self.log:
|
||||
self.c.log(
|
||||
"%s %s\r\n%s" % (
|
||||
direction, dst_str, cleanBin(contents)
|
||||
),
|
||||
"info"
|
||||
)
|
||||
dst.connection.send(contents)
|
||||
else:
|
||||
# socket.socket.send supports raw bytearrays/memoryviews
|
||||
self.c.log(
|
||||
"%s %s\r\n%s" % (
|
||||
direction,
|
||||
dst_str,
|
||||
cleanBin(buf.tobytes())
|
||||
),
|
||||
"debug"
|
||||
)
|
||||
if self.log:
|
||||
self.c.log(
|
||||
"%s %s\r\n%s" % (
|
||||
direction, dst_str, cleanBin(buf.tobytes())
|
||||
),
|
||||
"info"
|
||||
)
|
||||
dst.connection.send(buf[:size])
|
||||
except socket.error as e:
|
||||
self.c.log("TCP connection closed unexpectedly.", "debug")
|
||||
|
||||
@ -1,26 +1,54 @@
|
||||
from __future__ import absolute_import
|
||||
import os
|
||||
import re
|
||||
from netlib import http_auth, certutils
|
||||
from netlib import http_auth, certutils, tcp
|
||||
from .. import utils, platform, version
|
||||
from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode
|
||||
from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode
|
||||
|
||||
TRANSPARENT_SSL_PORTS = [443, 8443]
|
||||
CONF_BASENAME = "mitmproxy"
|
||||
CONF_DIR = "~/.mitmproxy"
|
||||
CA_DIR = "~/.mitmproxy"
|
||||
|
||||
|
||||
def parse_host_pattern(patterns):
|
||||
return [re.compile(p, re.IGNORECASE) for p in patterns]
|
||||
class HostMatcher(object):
|
||||
def __init__(self, patterns=[]):
|
||||
self.patterns = list(patterns)
|
||||
self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns]
|
||||
|
||||
def __call__(self, address):
|
||||
address = tcp.Address.wrap(address)
|
||||
host = "%s:%s" % (address.host, address.port)
|
||||
if any(rex.search(host) for rex in self.regexes):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def __nonzero__(self):
|
||||
return bool(self.patterns)
|
||||
|
||||
|
||||
class ProxyConfig:
|
||||
def __init__(self, host='', port=8080, server_version=version.NAMEVERSION,
|
||||
confdir=CONF_DIR, ca_file=None, clientcerts=None,
|
||||
no_upstream_cert=False, body_size_limit=None,
|
||||
mode=None, upstream_server=None, http_form_in=None, http_form_out=None,
|
||||
authenticator=None, ignore=[],
|
||||
ciphers=None, certs=[], certforward=False, ssl_ports=TRANSPARENT_SSL_PORTS):
|
||||
def __init__(
|
||||
self,
|
||||
host='',
|
||||
port=8080,
|
||||
server_version=version.NAMEVERSION,
|
||||
cadir=CA_DIR,
|
||||
clientcerts=None,
|
||||
no_upstream_cert=False,
|
||||
body_size_limit=None,
|
||||
mode=None,
|
||||
upstream_server=None,
|
||||
http_form_in=None,
|
||||
http_form_out=None,
|
||||
authenticator=None,
|
||||
ignore_hosts=[],
|
||||
tcp_hosts=[],
|
||||
ciphers=None,
|
||||
certs=[],
|
||||
certforward=False,
|
||||
ssl_ports=TRANSPARENT_SSL_PORTS
|
||||
):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.server_version = server_version
|
||||
@ -30,7 +58,9 @@ class ProxyConfig:
|
||||
self.body_size_limit = body_size_limit
|
||||
|
||||
if mode == "transparent":
|
||||
self.mode = TransparentProxyMode(platform.resolver(), TRANSPARENT_SSL_PORTS)
|
||||
self.mode = TransparentProxyMode(platform.resolver(), ssl_ports)
|
||||
elif mode == "socks5":
|
||||
self.mode = Socks5ProxyMode(ssl_ports)
|
||||
elif mode == "reverse":
|
||||
self.mode = ReverseProxyMode(upstream_server)
|
||||
elif mode == "upstream":
|
||||
@ -42,11 +72,11 @@ class ProxyConfig:
|
||||
self.mode.http_form_in = http_form_in or self.mode.http_form_in
|
||||
self.mode.http_form_out = http_form_out or self.mode.http_form_out
|
||||
|
||||
self.ignore = parse_host_pattern(ignore)
|
||||
self.check_ignore = HostMatcher(ignore_hosts)
|
||||
self.check_tcp = HostMatcher(tcp_hosts)
|
||||
self.authenticator = authenticator
|
||||
self.confdir = os.path.expanduser(confdir)
|
||||
self.ca_file = ca_file or os.path.join(self.confdir, CONF_BASENAME + "-ca.pem")
|
||||
self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME)
|
||||
self.cadir = os.path.expanduser(cadir)
|
||||
self.certstore = certutils.CertStore.from_store(self.cadir, CONF_BASENAME)
|
||||
for spec, cert in certs:
|
||||
self.certstore.add_cert_file(spec, cert)
|
||||
self.certforward = certforward
|
||||
@ -63,6 +93,9 @@ def process_proxy_options(parser, options):
|
||||
if not platform.resolver:
|
||||
return parser.error("Transparent mode not supported on this platform.")
|
||||
mode = "transparent"
|
||||
if options.socks_proxy:
|
||||
c += 1
|
||||
mode = "socks5"
|
||||
if options.reverse_proxy:
|
||||
c += 1
|
||||
mode = "reverse"
|
||||
@ -72,7 +105,7 @@ def process_proxy_options(parser, options):
|
||||
mode = "upstream"
|
||||
upstream_server = options.upstream_proxy
|
||||
if c > 1:
|
||||
return parser.error("Transparent mode, reverse mode and upstream proxy mode "
|
||||
return parser.error("Transparent, SOCKS5, reverse and upstream proxy mode "
|
||||
"are mutually exclusive.")
|
||||
|
||||
if options.clientcerts:
|
||||
@ -109,10 +142,16 @@ def process_proxy_options(parser, options):
|
||||
parser.error("Certificate file does not exist: %s" % parts[1])
|
||||
certs.append(parts)
|
||||
|
||||
ssl_ports = options.ssl_ports
|
||||
if options.ssl_ports != TRANSPARENT_SSL_PORTS:
|
||||
# arparse appends to default value by default, strip that off.
|
||||
# see http://bugs.python.org/issue16399
|
||||
ssl_ports = ssl_ports[len(TRANSPARENT_SSL_PORTS):]
|
||||
|
||||
return ProxyConfig(
|
||||
host=options.addr,
|
||||
port=options.port,
|
||||
confdir=options.confdir,
|
||||
cadir=options.cadir,
|
||||
clientcerts=options.clientcerts,
|
||||
no_upstream_cert=options.no_upstream_cert,
|
||||
body_size_limit=body_size_limit,
|
||||
@ -120,11 +159,13 @@ def process_proxy_options(parser, options):
|
||||
upstream_server=upstream_server,
|
||||
http_form_in=options.http_form_in,
|
||||
http_form_out=options.http_form_out,
|
||||
ignore=options.ignore,
|
||||
ignore_hosts=options.ignore_hosts,
|
||||
tcp_hosts=options.tcp_hosts,
|
||||
authenticator=authenticator,
|
||||
ciphers=options.ciphers,
|
||||
certs=certs,
|
||||
certforward=options.certforward,
|
||||
ssl_ports=ssl_ports
|
||||
)
|
||||
|
||||
|
||||
@ -133,10 +174,12 @@ def ssl_option_group(parser):
|
||||
group.add_argument(
|
||||
"--cert", dest='certs', default=[], type=str,
|
||||
metavar="SPEC", action="append",
|
||||
help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' \
|
||||
'The domain may include a wildcard, and is equal to "*" if not specified. ' \
|
||||
'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' \
|
||||
'it is used, else the default key in the conf dir is used. Can be passed multiple times.'
|
||||
help='Add an SSL certificate. SPEC is of the form "[domain=]path". '
|
||||
'The domain may include a wildcard, and is equal to "*" if not specified. '
|
||||
'The file at path is a certificate in PEM format. If a private key is included in the PEM, '
|
||||
'it is used, else the default key in the conf dir is used. '
|
||||
'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. '
|
||||
'Can be passed multiple times.'
|
||||
)
|
||||
group.add_argument(
|
||||
"--client-certs", action="store",
|
||||
@ -159,7 +202,7 @@ def ssl_option_group(parser):
|
||||
help="Don't connect to upstream server to look up certificate details."
|
||||
)
|
||||
group.add_argument(
|
||||
"--ssl-port", action="append", type=int, dest="ssl_ports", default=TRANSPARENT_SSL_PORTS,
|
||||
"--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS),
|
||||
metavar="PORT",
|
||||
help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. "
|
||||
"Defaults to %s." % str(TRANSPARENT_SSL_PORTS)
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
from __future__ import absolute_import
|
||||
|
||||
from netlib import socks
|
||||
|
||||
class ProxyError(Exception):
|
||||
def __init__(self, code, message, headers=None):
|
||||
@ -15,7 +15,7 @@ class ProxyMode(object):
|
||||
http_form_in = None
|
||||
http_form_out = None
|
||||
|
||||
def get_upstream_server(self, conn):
|
||||
def get_upstream_server(self, client_conn):
|
||||
"""
|
||||
Returns the address of the server to connect to.
|
||||
Returns None if the address needs to be determined on the protocol level (regular proxy mode)
|
||||
@ -46,7 +46,7 @@ class RegularProxyMode(ProxyMode):
|
||||
http_form_in = "absolute"
|
||||
http_form_out = "relative"
|
||||
|
||||
def get_upstream_server(self, conn):
|
||||
def get_upstream_server(self, client_conn):
|
||||
return None
|
||||
|
||||
|
||||
@ -58,9 +58,9 @@ class TransparentProxyMode(ProxyMode):
|
||||
self.resolver = resolver
|
||||
self.sslports = sslports
|
||||
|
||||
def get_upstream_server(self, conn):
|
||||
def get_upstream_server(self, client_conn):
|
||||
try:
|
||||
dst = self.resolver.original_addr(conn)
|
||||
dst = self.resolver.original_addr(client_conn.connection)
|
||||
except Exception, e:
|
||||
raise ProxyError(502, "Transparent mode failure: %s" % str(e))
|
||||
|
||||
@ -71,11 +71,80 @@ class TransparentProxyMode(ProxyMode):
|
||||
return [ssl, ssl] + list(dst)
|
||||
|
||||
|
||||
class Socks5ProxyMode(ProxyMode):
|
||||
http_form_in = "relative"
|
||||
http_form_out = "relative"
|
||||
|
||||
def __init__(self, sslports):
|
||||
self.sslports = sslports
|
||||
|
||||
@staticmethod
|
||||
def _assert_socks5(msg):
|
||||
if msg.ver != socks.VERSION.SOCKS5:
|
||||
if msg.ver == ord("G") and len(msg.methods) == ord("E"):
|
||||
guess = "Probably not a SOCKS request but a regular HTTP request. "
|
||||
else:
|
||||
guess = ""
|
||||
raise socks.SocksError(
|
||||
socks.REP.GENERAL_SOCKS_SERVER_FAILURE,
|
||||
guess + "Invalid SOCKS version. Expected 0x05, got 0x%x" % msg.ver)
|
||||
|
||||
def get_upstream_server(self, client_conn):
|
||||
try:
|
||||
# Parse Client Greeting
|
||||
client_greet = socks.ClientGreeting.from_file(client_conn.rfile)
|
||||
self._assert_socks5(client_greet)
|
||||
if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods:
|
||||
raise socks.SocksError(
|
||||
socks.METHOD.NO_ACCEPTABLE_METHODS,
|
||||
"mitmproxy only supports SOCKS without authentication"
|
||||
)
|
||||
|
||||
# Send Server Greeting
|
||||
server_greet = socks.ServerGreeting(
|
||||
socks.VERSION.SOCKS5,
|
||||
socks.METHOD.NO_AUTHENTICATION_REQUIRED
|
||||
)
|
||||
server_greet.to_file(client_conn.wfile)
|
||||
client_conn.wfile.flush()
|
||||
|
||||
# Parse Connect Request
|
||||
connect_request = socks.Message.from_file(client_conn.rfile)
|
||||
self._assert_socks5(connect_request)
|
||||
if connect_request.msg != socks.CMD.CONNECT:
|
||||
raise socks.SocksError(
|
||||
socks.REP.COMMAND_NOT_SUPPORTED,
|
||||
"mitmproxy only supports SOCKS5 CONNECT."
|
||||
)
|
||||
|
||||
# We do not connect here yet, as the clientconnect event has not been handled yet.
|
||||
|
||||
connect_reply = socks.Message(
|
||||
socks.VERSION.SOCKS5,
|
||||
socks.REP.SUCCEEDED,
|
||||
socks.ATYP.DOMAINNAME,
|
||||
client_conn.address # dummy value, we don't have an upstream connection yet.
|
||||
)
|
||||
connect_reply.to_file(client_conn.wfile)
|
||||
client_conn.wfile.flush()
|
||||
|
||||
ssl = bool(connect_request.addr.port in self.sslports)
|
||||
return ssl, ssl, connect_request.addr.host, connect_request.addr.port
|
||||
|
||||
except socks.SocksError as e:
|
||||
msg = socks.Message(5, e.code, socks.ATYP.DOMAINNAME, repr(e))
|
||||
try:
|
||||
msg.to_file(client_conn.wfile)
|
||||
except:
|
||||
pass
|
||||
raise ProxyError(502, "SOCKS5 mode failure: %s" % str(e))
|
||||
|
||||
|
||||
class _ConstDestinationProxyMode(ProxyMode):
|
||||
def __init__(self, dst):
|
||||
self.dst = dst
|
||||
|
||||
def get_upstream_server(self, conn):
|
||||
def get_upstream_server(self, client_conn):
|
||||
return self.dst
|
||||
|
||||
|
||||
|
||||
@ -70,13 +70,15 @@ class ConnectionHandler:
|
||||
|
||||
# Can we already identify the target server and connect to it?
|
||||
client_ssl, server_ssl = False, False
|
||||
upstream_info = self.config.mode.get_upstream_server(self.client_conn.connection)
|
||||
conn_kwargs = dict()
|
||||
upstream_info = self.config.mode.get_upstream_server(self.client_conn)
|
||||
if upstream_info:
|
||||
self.set_server_address(upstream_info[2:])
|
||||
client_ssl, server_ssl = upstream_info[:2]
|
||||
if self.check_ignore_address(self.server_conn.address):
|
||||
if self.config.check_ignore(self.server_conn.address):
|
||||
self.log("Ignore host: %s:%s" % self.server_conn.address(), "info")
|
||||
self.conntype = "tcp"
|
||||
conn_kwargs["log"] = False
|
||||
client_ssl, server_ssl = False, False
|
||||
else:
|
||||
pass # No upstream info from the metadata: upstream info in the protocol (e.g. HTTP absolute-form)
|
||||
@ -90,15 +92,18 @@ class ConnectionHandler:
|
||||
if client_ssl or server_ssl:
|
||||
self.establish_ssl(client=client_ssl, server=server_ssl)
|
||||
|
||||
# Delegate handling to the protocol handler
|
||||
protocol_handler(self.conntype)(self).handle_messages()
|
||||
if self.config.check_tcp(self.server_conn.address):
|
||||
self.log("Generic TCP mode for host: %s:%s" % self.server_conn.address(), "info")
|
||||
self.conntype = "tcp"
|
||||
|
||||
# Delegate handling to the protocol handler
|
||||
protocol_handler(self.conntype)(self, **conn_kwargs).handle_messages()
|
||||
|
||||
self.del_server_connection()
|
||||
self.log("clientdisconnect", "info")
|
||||
self.channel.tell("clientdisconnect", self)
|
||||
|
||||
except ProxyError as e:
|
||||
protocol_handler(self.conntype)(self).handle_error(e)
|
||||
protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e)
|
||||
except Exception:
|
||||
import traceback, sys
|
||||
|
||||
@ -106,6 +111,10 @@ class ConnectionHandler:
|
||||
print >> sys.stderr, traceback.format_exc()
|
||||
print >> sys.stderr, "mitmproxy has crashed!"
|
||||
print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy"
|
||||
finally:
|
||||
# Make sure that we close the server connection in any case.
|
||||
# The client connection is closed by the ProxyServer and does not have be handled here.
|
||||
self.del_server_connection()
|
||||
|
||||
def del_server_connection(self):
|
||||
"""
|
||||
@ -113,20 +122,13 @@ class ConnectionHandler:
|
||||
"""
|
||||
if self.server_conn and self.server_conn.connection:
|
||||
self.server_conn.finish()
|
||||
self.server_conn.close()
|
||||
self.log("serverdisconnect", "debug", ["%s:%s" % (self.server_conn.address.host,
|
||||
self.server_conn.address.port)])
|
||||
self.channel.tell("serverdisconnect", self)
|
||||
self.server_conn = None
|
||||
self.sni = None
|
||||
|
||||
def check_ignore_address(self, address):
|
||||
address = tcp.Address.wrap(address)
|
||||
host = "%s:%s" % (address.host, address.port)
|
||||
if host and any(rex.search(host) for rex in self.config.ignore):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def set_server_address(self, address):
|
||||
"""
|
||||
Sets a new server address with the given priority.
|
||||
@ -190,14 +192,14 @@ class ConnectionHandler:
|
||||
if client:
|
||||
if self.client_conn.ssl_established:
|
||||
raise ProxyError(502, "SSL to Client already established.")
|
||||
cert, key = self.find_cert()
|
||||
cert, key, chain_file = self.find_cert()
|
||||
try:
|
||||
self.client_conn.convert_to_ssl(
|
||||
cert, key,
|
||||
handle_sni=self.handle_sni,
|
||||
cipher_list=self.config.ciphers,
|
||||
dhparams=self.config.certstore.dhparams,
|
||||
ca_file=self.config.ca_file
|
||||
chain_file=chain_file
|
||||
)
|
||||
except tcp.NetLibError as v:
|
||||
raise ProxyError(400, repr(v))
|
||||
@ -234,7 +236,7 @@ class ConnectionHandler:
|
||||
|
||||
def find_cert(self):
|
||||
if self.config.certforward and self.server_conn.ssl_established:
|
||||
return self.server_conn.cert, self.config.certstore.gen_pkey(self.server_conn.cert)
|
||||
return self.server_conn.cert, self.config.certstore.gen_pkey(self.server_conn.cert), None
|
||||
else:
|
||||
host = self.server_conn.address.host
|
||||
sans = []
|
||||
@ -264,17 +266,17 @@ class ConnectionHandler:
|
||||
self.log("SNI received: %s" % self.sni, "debug")
|
||||
self.server_reconnect() # reconnect to upstream server with SNI
|
||||
# Now, change client context to reflect changed certificate:
|
||||
cert, key = self.find_cert()
|
||||
cert, key, chain_file = self.find_cert()
|
||||
new_context = self.client_conn._create_ssl_context(
|
||||
cert, key,
|
||||
method=SSL.TLSv1_METHOD,
|
||||
cipher_list=self.config.ciphers,
|
||||
dhparams=self.config.certstore.dhparams,
|
||||
ca_file=self.config.ca_file
|
||||
chain_file=chain_file
|
||||
)
|
||||
connection.set_context(new_context)
|
||||
# An unhandled exception in this method will core dump PyOpenSSL, so
|
||||
# make dang sure it doesn't happen.
|
||||
except Exception: # pragma: no cover
|
||||
except: # pragma: no cover
|
||||
import traceback
|
||||
self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error")
|
||||
self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error")
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
IVERSION = (0, 11)
|
||||
IVERSION = (0, 11, 1)
|
||||
VERSION = ".".join(str(i) for i in IVERSION)
|
||||
MINORVERSION = ".".join(str(i) for i in IVERSION[:2])
|
||||
NAME = "mitmproxy"
|
||||
NAMEVERSION = NAME + " " + VERSION
|
||||
|
||||
NEXT_MINORVERSION = list(IVERSION)
|
||||
NEXT_MINORVERSION[1] += 1
|
||||
NEXT_MINORVERSION = ".".join(str(i) for i in NEXT_MINORVERSION[:2])
|
||||
@ -93,7 +93,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": true
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -259,7 +259,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": true
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -425,7 +425,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": true
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -595,7 +595,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": true
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -765,7 +765,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": true
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -919,7 +919,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": false
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -1057,7 +1057,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": false
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -1195,7 +1195,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": false
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -1329,7 +1329,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": false
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -1483,7 +1483,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": false
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -1633,7 +1633,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": false
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -1767,7 +1767,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": false
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -1901,7 +1901,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": false
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
@ -2027,7 +2027,7 @@
|
||||
"clientcert": null,
|
||||
"ssl_established": false
|
||||
},
|
||||
"conntype": "http",
|
||||
"type": "http",
|
||||
"version": [
|
||||
0,
|
||||
11
|
||||
|
||||
@ -10,7 +10,8 @@
|
||||
# answer is to touch the __init__.py file in the zope directory. On my system:
|
||||
# touch /Library/Python/2.7/site-packages/zope/__init__.py
|
||||
|
||||
# To run, change into the pyinstaller directory, and then run this script.
|
||||
# To run, first install netlib and mitmproxy, then change into the pyinstaller
|
||||
# directory, and then run this script.
|
||||
|
||||
DST=/tmp/osx-mitmproxy
|
||||
MITMPROXY=~/mitmproxy/mitmproxy
|
||||
|
||||
@ -2,8 +2,6 @@
|
||||
- Bump the version number:
|
||||
|
||||
mitmproxy/libmproxy/version.py
|
||||
mitmproxy/requirements.txt
|
||||
mitmproxy/test/requirements.txt
|
||||
netlib/netlib/version.py
|
||||
netlib/requirements.txt
|
||||
netlib/test/requirements.txt
|
||||
@ -26,3 +24,22 @@
|
||||
- tar -xzvf pkgfile.tgz
|
||||
- virtualenv venv
|
||||
|
||||
- Build the OSX binaries
|
||||
- Follow instructions in osxbinaries
|
||||
- Package:
|
||||
cp -r ./doc /tmp/osx-mitmproxy/
|
||||
mv /tmp/osx-mitmproxy /tmp/osx-mitmproxy-VERSION
|
||||
tar -czvf /tmp/osx-mitmproxy-VERSION.tar.gz /tmp/osx-mitmproxy-VERSION
|
||||
mv /tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download
|
||||
|
||||
- Build the sources for each project:
|
||||
python ./setup.py sdist
|
||||
mv ./dist/FILE ~/mitmproxy/www.mitmproxy.org/src/download
|
||||
|
||||
|
||||
- Adjust links on www.mitmproxy.org
|
||||
|
||||
- Upload to pypi for each project:
|
||||
|
||||
python ./setup.py sdist upload
|
||||
|
||||
|
||||
@ -23,15 +23,16 @@ python ./setup.py -q sdist --dist-dir $DST
|
||||
echo "Creating virtualenv for test install..."
|
||||
virtualenv -q $DST/venv
|
||||
|
||||
cd $DST
|
||||
echo "Installing netlib..."
|
||||
$DST/venv/bin/pip -q install --download-cache ~/.pipcache $DST/netlib*
|
||||
./venv/bin/pip -q install --download-cache ~/.pipcache ./netlib*
|
||||
echo "Installing pathod..."
|
||||
$DST/venv/bin/pip -q install --download-cache ~/.pipcache $DST/pathod*
|
||||
./venv/bin/pip -q install --download-cache ~/.pipcache ./pathod*
|
||||
echo "Installing mitmproxy..."
|
||||
$DST/venv/bin/pip -q install --download-cache ~/.pipcache $DST/mitmproxy*
|
||||
./venv/bin/pip -q install --download-cache ~/.pipcache ./mitmproxy*
|
||||
|
||||
echo "Running binaries..."
|
||||
$DST/venv/bin/mitmproxy --version
|
||||
$DST/venv/bin/mitmdump --version
|
||||
$DST/venv/bin/pathod --version
|
||||
$DST/venv/bin/pathoc --version
|
||||
./venv/bin/mitmproxy --version
|
||||
./venv/bin/mitmdump --version
|
||||
./venv/bin/pathod --version
|
||||
./venv/bin/pathoc --version
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
-e git+https://github.com/mitmproxy/netlib.git#egg=netlib
|
||||
-e git+https://github.com/mitmproxy/pathod.git#egg=pathod
|
||||
-e .[dev]
|
||||
-e .[dev,examples]
|
||||
27
setup.py
27
setup.py
@ -16,13 +16,12 @@ if os.name != "nt":
|
||||
scripts.append("mitmproxy")
|
||||
|
||||
deps = {
|
||||
"netlib>=%s" % version.MINORVERSION,
|
||||
"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION),
|
||||
"pyasn1>0.1.2",
|
||||
"requests>=2.4.0",
|
||||
"pyOpenSSL>=0.14",
|
||||
"Flask>=0.10.1",
|
||||
"tornado>=4.0.2",
|
||||
"sortedcontainers>=0.9.1"
|
||||
"configargparse>=0.9.3"
|
||||
}
|
||||
script_deps = {
|
||||
"mitmproxy": {
|
||||
@ -37,10 +36,6 @@ for script in scripts:
|
||||
if os.name == "nt":
|
||||
deps.add("pydivert>=0.0.4") # Transparent proxying on Windows
|
||||
|
||||
console_scripts = [
|
||||
"%s = libmproxy.main:%s" % (s, s) for s in scripts
|
||||
]
|
||||
|
||||
|
||||
setup(
|
||||
name="mitmproxy",
|
||||
@ -67,14 +62,9 @@ setup(
|
||||
"Topic :: Internet :: Proxy Servers",
|
||||
"Topic :: Software Development :: Testing"
|
||||
],
|
||||
|
||||
packages=find_packages(),
|
||||
include_package_data=True,
|
||||
|
||||
entry_points={
|
||||
'console_scripts': console_scripts
|
||||
},
|
||||
|
||||
scripts = scripts,
|
||||
install_requires=list(deps),
|
||||
extras_require={
|
||||
'dev': [
|
||||
@ -82,12 +72,19 @@ setup(
|
||||
"nose>=1.3.0",
|
||||
"nose-cov>=1.6",
|
||||
"coveralls>=0.4.1",
|
||||
"pathod>=%s" % version.MINORVERSION
|
||||
"pathod>=%s, <%s" % (
|
||||
version.MINORVERSION, version.NEXT_MINORVERSION
|
||||
)
|
||||
],
|
||||
'contentviews': [
|
||||
"pyamf>=0.6.1",
|
||||
"protobuf>=2.5.0",
|
||||
"cssutils>=1.0"
|
||||
],
|
||||
'examples': [
|
||||
"pytz",
|
||||
"harparser",
|
||||
"beautifulsoup4"
|
||||
]
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
6
test/fuzzing/.env
Normal file
6
test/fuzzing/.env
Normal file
@ -0,0 +1,6 @@
|
||||
|
||||
MITMDUMP=../../mitmdump
|
||||
PATHOD=../../../pathod/pathod
|
||||
PATHOC=../../../pathod/pathoc
|
||||
FUZZ_SETTINGS=-remTt 1 -n 0
|
||||
|
||||
14
test/fuzzing/README
Normal file
14
test/fuzzing/README
Normal file
@ -0,0 +1,14 @@
|
||||
|
||||
A fuzzing architecture for mitmproxy
|
||||
====================================
|
||||
|
||||
Quick start:
|
||||
|
||||
honcho -f ./straight_stream start
|
||||
|
||||
|
||||
Notes:
|
||||
|
||||
- Processes are managed using honcho (pip install honcho)
|
||||
- Paths and common settings live in .env
|
||||
|
||||
4
test/fuzzing/client_patterns
Normal file
4
test/fuzzing/client_patterns
Normal file
@ -0,0 +1,4 @@
|
||||
get:'http://localhost:9999/p/200':ir,"\n"
|
||||
get:'http://localhost:9999/p/200':ir,"\0"
|
||||
get:'http://localhost:9999/p/200':ir,@5
|
||||
get:'http://localhost:9999/p/200':dr
|
||||
@ -3,20 +3,27 @@
|
||||
# mitmproxy/mitmdump is running on port 8080 in straight proxy mode.
|
||||
# pathod is running on port 9999
|
||||
|
||||
BASE_HTTP="/Users/aldo/git/public/pathod/pathoc -Tt 1 -eo -I 200,400,405,502 -p 8080 localhost "
|
||||
BASE="../../../"
|
||||
BASE_HTTP=$BASE"/pathod/pathoc -Tt 1 -e -I 200,400,405,502 -p 8080 localhost "
|
||||
BASE_HTTPS=$BASE"/pathod/pathoc -sc localhost:9999 -Tt 1 -eo -I 200,400,404,405,502,800 -p 8080 localhost "
|
||||
|
||||
#$BASE_HTTP -n 10000 "get:'http://localhost:9999':ir,@1"
|
||||
#$BASE_HTTP -n 100 "get:'http://localhost:9999':dr"
|
||||
#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200:ir,@300.0
|
||||
#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200':ir,@300"
|
||||
|
||||
#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200:ir,@1'"
|
||||
#$BASE_HTTP -n 100 "get:'http://localhost:9999/p/200:dr'"
|
||||
#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200:ir,@100'"
|
||||
|
||||
|
||||
# Assuming:
|
||||
# mitmproxy/mitmdump is running on port 8080 in straight proxy mode.
|
||||
# pathod with SSL enabled is running on port 9999
|
||||
|
||||
BASE_HTTPS="/Users/aldo/git/public/pathod/pathoc -sc localhost:9999 -Tt 1 -eo -I 200,400,404,405,502,800 -p 8080 localhost "
|
||||
$BASE_HTTPS -en 10000 "get:'/p/200:b@10:ir,@1'"
|
||||
#$BASE_HTTPS -en 10000 "get:'/p/200:b@100:ir,@1'"
|
||||
#$BASE_HTTPS -en 10000 "get:'/p/200:ir,@1'"
|
||||
|
||||
#$BASE_HTTPS -n 100 "get:'/p/200:dr'"
|
||||
#$BASE_HTTPS -n 10000 "get:'/p/200:ir,@3000'"
|
||||
#$BASE_HTTPS -n 10000 "get:'/p/200:ir,\"\\n\"'"
|
||||
|
||||
|
||||
9
test/fuzzing/reverse_patterns
Normal file
9
test/fuzzing/reverse_patterns
Normal file
@ -0,0 +1,9 @@
|
||||
get:'/p/200':b@10:ir,"\n"
|
||||
get:'/p/200':b@10:ir,"\r\n"
|
||||
get:'/p/200':b@10:ir,"\0"
|
||||
get:'/p/200':b@10:ir,@5
|
||||
get:'/p/200':b@10:dr
|
||||
|
||||
get:'/p/200:b@10:ir,@1'
|
||||
get:'/p/200:b@10:dr'
|
||||
get:'/p/200:b@10:ir,@100'
|
||||
6
test/fuzzing/straight_stream
Normal file
6
test/fuzzing/straight_stream
Normal file
@ -0,0 +1,6 @@
|
||||
|
||||
mitmdump: $MITMDUMP
|
||||
pathod: $PATHOD
|
||||
pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns
|
||||
#pathoc: sleep 2 && $PATHOC localhost:8080 /tmp/err
|
||||
|
||||
17
test/fuzzing/straight_stream_patterns
Normal file
17
test/fuzzing/straight_stream_patterns
Normal file
@ -0,0 +1,17 @@
|
||||
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'\n'
|
||||
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'a'
|
||||
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'9'
|
||||
get:'http://localhost:9999/p/':s'200:b"foo"':ir,':'
|
||||
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'"'
|
||||
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'-'
|
||||
|
||||
get:'http://localhost:9999/p/':s'200:b"foo":ir,"\n"'
|
||||
get:'http://localhost:9999/p/':s'200:b"foo":ir,"a"'
|
||||
get:'http://localhost:9999/p/':s'200:b"foo":ir,"9"'
|
||||
get:'http://localhost:9999/p/':s'200:b"foo":ir,":"'
|
||||
get:'http://localhost:9999/p/':s"200:b'foo':ir,'\"'"
|
||||
get:'http://localhost:9999/p/':s'200:b"foo":ir,"-"'
|
||||
get:'http://localhost:9999/p/':s'200:b"foo":dr'
|
||||
|
||||
get:'http://localhost:9999/p/':s'200:b"foo"':ir,@2
|
||||
get:'http://localhost:9999/p/':s'200:b"foo":ir,@2'
|
||||
6
test/fuzzing/straight_stream_ssl
Normal file
6
test/fuzzing/straight_stream_ssl
Normal file
@ -0,0 +1,6 @@
|
||||
|
||||
mitmdump: $MITMDUMP -q --stream 1
|
||||
pathod: $PATHOD
|
||||
pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns
|
||||
#pathoc: sleep 2 && $PATHOC localhost:8080 /tmp/err
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
import argparse
|
||||
from libmproxy import cmdline
|
||||
import tutils
|
||||
import os.path
|
||||
|
||||
|
||||
def test_parse_replace_hook():
|
||||
@ -51,6 +50,7 @@ def test_parse_setheaders():
|
||||
x = cmdline.parse_setheader("/foo/bar/voing")
|
||||
assert x == ("foo", "bar", "voing")
|
||||
|
||||
|
||||
def test_common():
|
||||
parser = argparse.ArgumentParser()
|
||||
cmdline.common_options(parser)
|
||||
@ -108,3 +108,19 @@ def test_common():
|
||||
assert len(v) == 1
|
||||
assert v[0][2].strip() == "replacecontents"
|
||||
|
||||
|
||||
def test_mitmproxy():
|
||||
ap = cmdline.mitmproxy()
|
||||
assert ap
|
||||
|
||||
|
||||
def test_mitmdump():
|
||||
ap = cmdline.mitmdump()
|
||||
assert ap
|
||||
|
||||
|
||||
def test_mitmweb():
|
||||
ap = cmdline.mitmweb()
|
||||
assert ap
|
||||
|
||||
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
import os
|
||||
from cStringIO import StringIO
|
||||
from libmproxy import dump, flow, proxy
|
||||
from libmproxy import dump, flow
|
||||
from libmproxy.protocol import http
|
||||
from libmproxy.proxy.primitives import Log
|
||||
import tutils
|
||||
import mock
|
||||
|
||||
|
||||
def test_strfuncs():
|
||||
t = tutils.tresp()
|
||||
t.is_replay = True
|
||||
@ -58,6 +60,18 @@ class TestDumpMaster:
|
||||
assert m.handle_error(f)
|
||||
assert "error" in cs.getvalue()
|
||||
|
||||
def test_missing_content(self):
|
||||
cs = StringIO()
|
||||
o = dump.Options(flow_detail=3)
|
||||
m = dump.DumpMaster(None, o, outfile=cs)
|
||||
f = tutils.tflow()
|
||||
f.request.content = http.CONTENT_MISSING
|
||||
m.handle_request(f)
|
||||
f.response = tutils.tresp()
|
||||
f.response.content = http.CONTENT_MISSING
|
||||
m.handle_response(f)
|
||||
assert "content missing" in cs.getvalue()
|
||||
|
||||
def test_replay(self):
|
||||
cs = StringIO()
|
||||
|
||||
|
||||
@ -1,10 +1,8 @@
|
||||
import glob
|
||||
import mock
|
||||
from libmproxy import utils, script
|
||||
from libmproxy.proxy import config
|
||||
import tservers
|
||||
|
||||
@mock.patch.dict("sys.modules", {"bs4": mock.Mock()})
|
||||
def test_load_scripts():
|
||||
example_dir = utils.Data("libmproxy").path("../examples")
|
||||
scripts = glob.glob("%s/*.py" % example_dir)
|
||||
@ -12,8 +10,11 @@ def test_load_scripts():
|
||||
tmaster = tservers.TestMaster(config.ProxyConfig())
|
||||
|
||||
for f in scripts:
|
||||
if "har_extractor" in f:
|
||||
f += " foo"
|
||||
if "iframe_injector" in f:
|
||||
f += " foo" # one argument required
|
||||
if "modify_response_body" in f:
|
||||
f += " foo bar" # two arguments required
|
||||
script.Script(f, tmaster) # Loads the script file.
|
||||
s = script.Script(f, tmaster) # Loads the script file.
|
||||
s.unload()
|
||||
@ -5,8 +5,10 @@ import mock
|
||||
from libmproxy import filt, protocol, controller, utils, tnetstring, flow
|
||||
from libmproxy.protocol.primitives import Error, Flow
|
||||
from libmproxy.protocol.http import decoded, CONTENT_MISSING
|
||||
from libmproxy.proxy.connection import ClientConnection, ServerConnection
|
||||
from netlib import tcp
|
||||
from libmproxy.proxy.config import HostMatcher
|
||||
from libmproxy.proxy import ProxyConfig
|
||||
from libmproxy.proxy.server import DummyServer
|
||||
from libmproxy.proxy.connection import ClientConnection
|
||||
import tutils
|
||||
|
||||
|
||||
@ -84,19 +86,20 @@ class TestClientPlaybackState:
|
||||
fm = flow.FlowMaster(None, s)
|
||||
fm.start_client_playback([first, tutils.tflow()], True)
|
||||
c = fm.client_playback
|
||||
c.testing = True
|
||||
|
||||
assert not c.done()
|
||||
assert not s.flow_count()
|
||||
assert c.count() == 2
|
||||
c.tick(fm, testing=True)
|
||||
c.tick(fm)
|
||||
assert s.flow_count()
|
||||
assert c.count() == 1
|
||||
|
||||
c.tick(fm, testing=True)
|
||||
c.tick(fm)
|
||||
assert c.count() == 1
|
||||
|
||||
c.clear(c.current)
|
||||
c.tick(fm, testing=True)
|
||||
c.tick(fm)
|
||||
assert c.count() == 0
|
||||
c.clear(c.current)
|
||||
assert c.done()
|
||||
@ -531,6 +534,14 @@ class TestSerialize:
|
||||
fm.load_flows(r)
|
||||
assert len(s._flow_list) == 6
|
||||
|
||||
def test_load_flows_reverse(self):
|
||||
r = self._treader()
|
||||
s = flow.State()
|
||||
conf = ProxyConfig(mode="reverse", upstream_server=[True,True,"use-this-domain",80])
|
||||
fm = flow.FlowMaster(DummyServer(conf), s)
|
||||
fm.load_flows(r)
|
||||
assert s._flow_list[0].request.host == "use-this-domain"
|
||||
|
||||
def test_filter(self):
|
||||
sio = StringIO()
|
||||
fl = filt.parse("~c 200")
|
||||
@ -584,11 +595,11 @@ class TestFlowMaster:
|
||||
|
||||
def test_getset_ignore(self):
|
||||
p = mock.Mock()
|
||||
p.config.ignore = []
|
||||
p.config.check_ignore = HostMatcher()
|
||||
fm = flow.FlowMaster(p, flow.State())
|
||||
assert not fm.get_ignore()
|
||||
fm.set_ignore(["^apple\.com:", ":443$"])
|
||||
assert fm.get_ignore()
|
||||
assert not fm.get_ignore_filter()
|
||||
fm.set_ignore_filter(["^apple\.com:", ":443$"])
|
||||
assert fm.get_ignore_filter()
|
||||
|
||||
def test_replay(self):
|
||||
s = flow.State()
|
||||
@ -600,6 +611,9 @@ class TestFlowMaster:
|
||||
f.intercepting = True
|
||||
assert "intercepting" in fm.replay_request(f)
|
||||
|
||||
f.live = True
|
||||
assert "live" in fm.replay_request(f)
|
||||
|
||||
def test_script_reqerr(self):
|
||||
s = flow.State()
|
||||
fm = flow.FlowMaster(None, s)
|
||||
@ -679,9 +693,11 @@ class TestFlowMaster:
|
||||
|
||||
f = tutils.tflow(resp=True)
|
||||
pb = [tutils.tflow(resp=True), f]
|
||||
fm = flow.FlowMaster(None, s)
|
||||
|
||||
fm = flow.FlowMaster(DummyServer(ProxyConfig()), s)
|
||||
assert not fm.start_server_playback(pb, False, [], False, False, None, False)
|
||||
assert not fm.start_client_playback(pb, False)
|
||||
fm.client_playback.testing = True
|
||||
|
||||
q = Queue.Queue()
|
||||
assert not fm.state.flow_count()
|
||||
|
||||
@ -23,7 +23,7 @@ def test_stripped_chunked_encoding_no_content():
|
||||
|
||||
|
||||
class TestHTTPRequest:
|
||||
def test_asterisk_form(self):
|
||||
def test_asterisk_form_in(self):
|
||||
s = StringIO("OPTIONS * HTTP/1.1")
|
||||
f = tutils.tflow(req=None)
|
||||
f.request = HTTPRequest.from_stream(s)
|
||||
@ -31,9 +31,11 @@ class TestHTTPRequest:
|
||||
f.request.host = f.server_conn.address.host
|
||||
f.request.port = f.server_conn.address.port
|
||||
f.request.scheme = "http"
|
||||
assert f.request.assemble() == "OPTIONS * HTTP/1.1\r\nHost: address:22\r\n\r\n"
|
||||
assert f.request.assemble() == ("OPTIONS * HTTP/1.1\r\n"
|
||||
"Host: address:22\r\n"
|
||||
"Content-Length: 0\r\n\r\n")
|
||||
|
||||
def test_origin_form(self):
|
||||
def test_relative_form_in(self):
|
||||
s = StringIO("GET /foo\xff HTTP/1.1")
|
||||
tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s)
|
||||
s = StringIO("GET /foo HTTP/1.1\r\nConnection: Upgrade\r\nUpgrade: h2c")
|
||||
@ -52,22 +54,47 @@ class TestHTTPRequest:
|
||||
r.update_host_header()
|
||||
assert "Host" in r.headers
|
||||
|
||||
|
||||
def test_authority_form(self):
|
||||
def test_authority_form_in(self):
|
||||
s = StringIO("CONNECT oops-no-port.com HTTP/1.1")
|
||||
tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s)
|
||||
s = StringIO("CONNECT address:22 HTTP/1.1")
|
||||
r = HTTPRequest.from_stream(s)
|
||||
r.scheme, r.host, r.port = "http", "address", 22
|
||||
assert r.assemble() == "CONNECT address:22 HTTP/1.1\r\nHost: address:22\r\n\r\n"
|
||||
assert r.assemble() == ("CONNECT address:22 HTTP/1.1\r\n"
|
||||
"Host: address:22\r\n"
|
||||
"Content-Length: 0\r\n\r\n")
|
||||
assert r.pretty_url(False) == "address:22"
|
||||
|
||||
def test_absolute_form(self):
|
||||
def test_absolute_form_in(self):
|
||||
s = StringIO("GET oops-no-protocol.com HTTP/1.1")
|
||||
tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s)
|
||||
s = StringIO("GET http://address:22/ HTTP/1.1")
|
||||
r = HTTPRequest.from_stream(s)
|
||||
assert r.assemble() == "GET http://address:22/ HTTP/1.1\r\nHost: address:22\r\n\r\n"
|
||||
assert r.assemble() == "GET http://address:22/ HTTP/1.1\r\nHost: address:22\r\nContent-Length: 0\r\n\r\n"
|
||||
|
||||
def test_http_options_relative_form_in(self):
|
||||
"""
|
||||
Exercises fix for Issue #392.
|
||||
"""
|
||||
s = StringIO("OPTIONS /secret/resource HTTP/1.1")
|
||||
r = HTTPRequest.from_stream(s)
|
||||
r.host = 'address'
|
||||
r.port = 80
|
||||
r.scheme = "http"
|
||||
assert r.assemble() == ("OPTIONS /secret/resource HTTP/1.1\r\n"
|
||||
"Host: address\r\n"
|
||||
"Content-Length: 0\r\n\r\n")
|
||||
|
||||
def test_http_options_absolute_form_in(self):
|
||||
s = StringIO("OPTIONS http://address/secret/resource HTTP/1.1")
|
||||
r = HTTPRequest.from_stream(s)
|
||||
r.host = 'address'
|
||||
r.port = 80
|
||||
r.scheme = "http"
|
||||
assert r.assemble() == ("OPTIONS http://address:80/secret/resource HTTP/1.1\r\n"
|
||||
"Host: address\r\n"
|
||||
"Content-Length: 0\r\n\r\n")
|
||||
|
||||
|
||||
def test_assemble_unknown_form(self):
|
||||
r = tutils.treq()
|
||||
@ -133,4 +160,4 @@ class TestInvalidRequests(tservers.HTTPProxTest):
|
||||
p.connect()
|
||||
r = p.request("get:/p/200")
|
||||
assert r.status_code == 400
|
||||
assert "Invalid HTTP request form" in r.content
|
||||
assert "Invalid HTTP request form" in r.content
|
||||
|
||||
@ -70,9 +70,9 @@ class TestProcessProxyOptions:
|
||||
def test_simple(self):
|
||||
assert self.p()
|
||||
|
||||
def test_confdir(self):
|
||||
with tutils.tmpdir() as confdir:
|
||||
self.assert_noerr("--confdir", confdir)
|
||||
def test_cadir(self):
|
||||
with tutils.tmpdir() as cadir:
|
||||
self.assert_noerr("--cadir", cadir)
|
||||
|
||||
@mock.patch("libmproxy.platform.resolver", None)
|
||||
def test_no_transparent(self):
|
||||
@ -94,12 +94,12 @@ class TestProcessProxyOptions:
|
||||
self.assert_err("mutually exclusive", "-R", "http://localhost", "-T")
|
||||
|
||||
def test_client_certs(self):
|
||||
with tutils.tmpdir() as confdir:
|
||||
self.assert_noerr("--client-certs", confdir)
|
||||
with tutils.tmpdir() as cadir:
|
||||
self.assert_noerr("--client-certs", cadir)
|
||||
self.assert_err("directory does not exist", "--client-certs", "nonexistent")
|
||||
|
||||
def test_certs(self):
|
||||
with tutils.tmpdir() as confdir:
|
||||
with tutils.tmpdir() as cadir:
|
||||
self.assert_noerr("--cert", tutils.test_data.path("data/testkey.pem"))
|
||||
self.assert_err("does not exist", "--cert", "nonexistent")
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import socket, time
|
||||
from libmproxy.proxy.config import parse_host_pattern
|
||||
from libmproxy.proxy.config import HostMatcher
|
||||
from netlib import tcp, http_auth, http
|
||||
from libpathod import pathoc, pathod
|
||||
from netlib.certutils import SSLCert
|
||||
@ -19,6 +19,17 @@ class CommonMixin:
|
||||
def test_large(self):
|
||||
assert len(self.pathod("200:b@50k").content) == 1024*50
|
||||
|
||||
@staticmethod
|
||||
def wait_until_not_live(flow):
|
||||
"""
|
||||
Race condition: We don't want to replay the flow while it is still live.
|
||||
"""
|
||||
s = time.time()
|
||||
while flow.live:
|
||||
time.sleep(0.001)
|
||||
if time.time() - s > 5:
|
||||
raise RuntimeError("Flow is live for too long.")
|
||||
|
||||
def test_replay(self):
|
||||
assert self.pathod("304").status_code == 304
|
||||
if isinstance(self, tservers.HTTPUpstreamProxTest) and self.ssl:
|
||||
@ -28,6 +39,7 @@ class CommonMixin:
|
||||
l = self.master.state.view[-1]
|
||||
assert l.response.code == 304
|
||||
l.request.path = "/p/305"
|
||||
self.wait_until_not_live(l)
|
||||
rt = self.master.replay_request(l, block=True)
|
||||
assert l.response.code == 305
|
||||
|
||||
@ -79,11 +91,14 @@ class CommonMixin:
|
||||
|
||||
class TcpMixin:
|
||||
def _ignore_on(self):
|
||||
ignore = parse_host_pattern([".+:%s" % self.server.port])[0]
|
||||
self.config.ignore.append(ignore)
|
||||
assert not hasattr(self, "_ignore_backup")
|
||||
self._ignore_backup = self.config.check_ignore
|
||||
self.config.check_ignore = HostMatcher([".+:%s" % self.server.port] + self.config.check_ignore.patterns)
|
||||
|
||||
def _ignore_off(self):
|
||||
self.config.ignore.pop()
|
||||
assert hasattr(self, "_ignore_backup")
|
||||
self.config.check_ignore = self._ignore_backup
|
||||
del self._ignore_backup
|
||||
|
||||
def test_ignore(self):
|
||||
spec = '304:h"Alternate-Protocol"="mitmproxy-will-remove-this"'
|
||||
@ -114,6 +129,40 @@ class TcpMixin:
|
||||
tutils.raises("invalid server response", self.pathod, spec) # pathoc tries to parse answer as HTTP
|
||||
self._ignore_off()
|
||||
|
||||
def _tcpproxy_on(self):
|
||||
assert not hasattr(self, "_tcpproxy_backup")
|
||||
self._tcpproxy_backup = self.config.check_tcp
|
||||
self.config.check_tcp = HostMatcher([".+:%s" % self.server.port] + self.config.check_tcp.patterns)
|
||||
|
||||
def _tcpproxy_off(self):
|
||||
assert hasattr(self, "_tcpproxy_backup")
|
||||
self.config.check_ignore = self._tcpproxy_backup
|
||||
del self._tcpproxy_backup
|
||||
|
||||
|
||||
def test_tcp(self):
|
||||
spec = '304:h"Alternate-Protocol"="mitmproxy-will-remove-this"'
|
||||
n = self.pathod(spec)
|
||||
self._tcpproxy_on()
|
||||
i = self.pathod(spec)
|
||||
i2 = self.pathod(spec)
|
||||
self._tcpproxy_off()
|
||||
|
||||
assert i.status_code == i2.status_code == n.status_code == 304
|
||||
assert "Alternate-Protocol" in i.headers
|
||||
assert "Alternate-Protocol" in i2.headers
|
||||
assert "Alternate-Protocol" not in n.headers
|
||||
|
||||
# Test that we get the original SSL cert
|
||||
if self.ssl:
|
||||
i_cert = SSLCert(i.sslinfo.certchain[0])
|
||||
i2_cert = SSLCert(i2.sslinfo.certchain[0])
|
||||
n_cert = SSLCert(n.sslinfo.certchain[0])
|
||||
|
||||
assert i_cert == i2_cert == n_cert
|
||||
|
||||
# Make sure that TCP messages are in the event log.
|
||||
assert any("mitmproxy-will-remove-this" in m for m in self.master.log)
|
||||
|
||||
class AppMixin:
|
||||
def test_app(self):
|
||||
@ -579,16 +628,50 @@ class TestUpstreamProxy(tservers.HTTPUpstreamProxTest, CommonMixin, AppMixin):
|
||||
class TestUpstreamProxySSL(tservers.HTTPUpstreamProxTest, CommonMixin, TcpMixin):
|
||||
ssl = True
|
||||
|
||||
def _host_pattern_on(self, attr):
|
||||
"""
|
||||
Updates config.check_tcp or check_ignore, depending on attr.
|
||||
"""
|
||||
assert not hasattr(self, "_ignore_%s_backup" % attr)
|
||||
backup = []
|
||||
for proxy in self.chain:
|
||||
old_matcher = getattr(proxy.tmaster.server.config, "check_%s" % attr)
|
||||
backup.append(old_matcher)
|
||||
setattr(
|
||||
proxy.tmaster.server.config,
|
||||
"check_%s" % attr,
|
||||
HostMatcher([".+:%s" % self.server.port] + old_matcher.patterns)
|
||||
)
|
||||
|
||||
setattr(self, "_ignore_%s_backup" % attr, backup)
|
||||
|
||||
def _host_pattern_off(self, attr):
|
||||
backup = getattr(self, "_ignore_%s_backup" % attr)
|
||||
for proxy in reversed(self.chain):
|
||||
setattr(
|
||||
proxy.tmaster.server.config,
|
||||
"check_%s" % attr,
|
||||
backup.pop()
|
||||
)
|
||||
|
||||
assert not backup
|
||||
delattr(self, "_ignore_%s_backup" % attr)
|
||||
|
||||
def _ignore_on(self):
|
||||
super(TestUpstreamProxySSL, self)._ignore_on()
|
||||
ignore = parse_host_pattern([".+:%s" % self.server.port])[0]
|
||||
for proxy in self.chain:
|
||||
proxy.tmaster.server.config.ignore.append(ignore)
|
||||
self._host_pattern_on("ignore")
|
||||
|
||||
def _ignore_off(self):
|
||||
super(TestUpstreamProxySSL, self)._ignore_off()
|
||||
for proxy in self.chain:
|
||||
proxy.tmaster.server.config.ignore.pop()
|
||||
self._host_pattern_off("ignore")
|
||||
|
||||
def _tcpproxy_on(self):
|
||||
super(TestUpstreamProxySSL, self)._tcpproxy_on()
|
||||
self._host_pattern_on("tcp")
|
||||
|
||||
def _tcpproxy_off(self):
|
||||
super(TestUpstreamProxySSL, self)._tcpproxy_off()
|
||||
self._host_pattern_off("tcp")
|
||||
|
||||
def test_simple(self):
|
||||
p = self.pathoc()
|
||||
|
||||
21
test/tools/passive_close.py
Normal file
21
test/tools/passive_close.py
Normal file
@ -0,0 +1,21 @@
|
||||
import SocketServer
|
||||
from threading import Thread
|
||||
from time import sleep
|
||||
|
||||
class service(SocketServer.BaseRequestHandler):
|
||||
def handle(self):
|
||||
data = 'dummy'
|
||||
print "Client connected with ", self.client_address
|
||||
while True:
|
||||
self.request.send("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 7\r\n\r\ncontent")
|
||||
data = self.request.recv(1024)
|
||||
if not len(data):
|
||||
print "Connection closed by remote: ", self.client_address
|
||||
sleep(3600)
|
||||
|
||||
|
||||
class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
|
||||
pass
|
||||
|
||||
server = ThreadedTCPServer(('',1520), service)
|
||||
server.serve_forever()
|
||||
@ -99,7 +99,7 @@ class ProxTestBase(object):
|
||||
|
||||
@classmethod
|
||||
def teardownAll(cls):
|
||||
shutil.rmtree(cls.confdir)
|
||||
shutil.rmtree(cls.cadir)
|
||||
cls.proxy.shutdown()
|
||||
cls.server.shutdown()
|
||||
cls.server2.shutdown()
|
||||
@ -116,10 +116,10 @@ class ProxTestBase(object):
|
||||
|
||||
@classmethod
|
||||
def get_proxy_config(cls):
|
||||
cls.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy")
|
||||
cls.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy")
|
||||
return dict(
|
||||
no_upstream_cert = cls.no_upstream_cert,
|
||||
confdir = cls.confdir,
|
||||
cadir = cls.cadir,
|
||||
authenticator = cls.authenticator,
|
||||
certforward = cls.certforward,
|
||||
ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user