Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-23 00:01:36 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of github.com:mitmproxy/mitmproxy

Browse Source
This commit is contained in:
Maximilian Hils 2015-05-31 01:51:31 +02:00
commit ec92fca8c0
122 changed files with 8494 additions and 1006 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.env
View File

@ -1,5 +1,5 @@
DIR=`dirname $0`
if [ -z "$VIRTUAL_ENV" ] && [ -f $DIR/../venv.mitmproxy/bin/activate ]; then
DIR="${0%/*}"
if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then
echo "Activating mitmproxy virtualenv..."
source $DIR/../venv.mitmproxy/bin/activate
source "$DIR/../venv.mitmproxy/bin/activate"
fi

20
check_coding_style.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
autopep8 -i -r -a -a .
if [[ -n "$(git status -s)" ]]; then
echo "autopep8 yielded the following changes:"
git status -s
git --no-pager diff
exit 1
fi
autoflake -i -r --remove-all-unused-imports --remove-unused-variables .
if [[ -n "$(git status -s)" ]]; then
echo "autoflake yielded the following changes:"
git status -s
git --no-pager diff
exit 1
fi
echo "Coding style seems to be ok."
exit 0

9
doc-src/01-bootstrap.min.css vendored
View File

File diff suppressed because one or more lines are too long

6706
doc-src/01-vendor.css Normal file
View File

File diff suppressed because it is too large Load Diff

43
doc-src/02-app.css Normal file
View File

@ -0,0 +1,43 @@
.masthead {
text-align: center;
border-bottom: 0;
}
.frontpage .talks div {
margin-bottom: 10px;
}
.nav-sidebar {
background-color: #f0f0f0;
margin-bottom: 20px;
}
.nav-sidebar li {
line-height: 1.1;
}
.nav-sidebar li > a,
.nav-sidebar .nav-header {
padding-left: 20px;
}
.nav-sidebar .nav-header {
margin-top: 1em;
font-size: 1.2em;
font-weight: bold;
}
.nav-sidebar .active > a,
.nav-sidebar .active > a:hover,
.nav-sidebar .active > a:focus {
color: #fff;
background-color: #428bca;
}
.tablenum {
font-weight: bold;
}
.nowrap {
white-space: nowrap;
}
.page-header {
margin: 0px 0 22px;
}
.page-header h1 {
margin-top: 0px;
}
/*# sourceMappingURL=02-app.css.map */

20
doc-src/02-docstyle.css
View File

@ -1,20 +0,0 @@
body {
padding-top: 60px;
padding-bottom: 40px;
}
.tablenum {
font-weight: bold;
}
.nowrap {
white-space: nowrap;
}
h1 {
line-height: 1.1;
}
.page-header {
margin: 0px 0 22px;
}

40
doc-src/_layout.html
View File

@ -1,36 +1,44 @@
<div class="navbar navbar-fixed-top">
<div class="navbar-inner">
<!DOCTYPE html>
<html>
<head>
<title>@!pageTitle!@</title>
$!header!$
</head>
<body>
<div class="navbar navbar-default navbar-static-top">
<div class="container">
<a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<div class="navbar-header">
<a class="navbar-brand" href="@!urlTo("/index.html")!@">
<img height="20px" src="@!urlTo("mitmproxy-long.png")!@"/>
</a>
<a class="brand" href="@!urlTo(idxpath)!@">mitmproxy $!VERSION!$ docs</a>
</div><!--/.nav-collapse -->
</div>
<div class="navbar-header navbar-right">
<a class="navbar-brand" hre="#">$!VERSION!$ docs</a>
</div>
</div>
</div>
</div>
<div class="container">
<div class="container">
<div class="row">
<div class="span3">
<div class="well sidebar-nav">
<div class="col-md-3">
$!navbar!$
</div>
</div>
<div class="span9">
<div class="col-md-9">
<div class="page-header">
<h1>@!this.title!@</h1>
</div>
$!body!$
</div>
</div>
</div>
<div class="container">
<hr>
<footer>
<p>@!copyright!@</p>
</footer>
</div>
</div>
</body>
</html>

2
doc-src/_nav.html
View File

@ -1,4 +1,4 @@
<ul class="nav nav-list">
<ul class="nav nav-sidebar">
$!nav(idxpath, this, state)!$
$!nav("install.html", this, state)!$
$!nav("certinstall.html", this, state)!$

42
doc-src/_websitelayout.html
View File

@ -1,42 +0,0 @@
<div class="navbar navbar-fixed-top">
<div class="navbar-inner">
<div class="container">
<a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</a>
<a class="brand" href="@!urlTo(idxpath)!@">mitmproxy</a>
<div class="nav">
<ul class="nav">
<li $!'class="active"' if this.match("/index.html", True) else ""!$> <a href="@!top!@/index.html">home</a> </li>
<li $!'class="active"' if this.under("/doc") else ""!$><a href="@!top!@/doc/index.html">docs</a></li>
<li $!'class="active"' if this.under("/about.html") else ""!$><a href="@!top!@/about.html">about</a></li>
</ul>
</div>
</div>
</div>
</div>
<div class="container">
<div class="row">
<div class="span3">
<div class="well sidebar-nav">
$!navbar!$
</div>
</div>
<div class="span9">
<div class="page-header">
<h1>@!this.title!@</h1>
</div>
$!body!$
</div>
</div>
<hr>
<footer>
<p>@!copyright!@</p>
</footer>
</div>

6
doc-src/dev/architecture.html
View File

@ -1,7 +1,7 @@
To give you a better understanding of how mitmproxy works, mitmproxy's high-level architecture is detailed
in the following graphic:
To give you a better understanding of how mitmproxy works, mitmproxy's
high-level architecture is detailed in the following graphic:
<img src="@!urlTo('schematics/architecture.png')!@">
<img class="img-responsive" src="@!urlTo('schematics/architecture.png')!@">
<a href="@!urlTo('schematics/architecture.pdf')!@">(architecture.pdf)</a>
<p>Please don't refrain from asking any further

2
doc-src/dev/index.py
View File

@ -4,5 +4,5 @@ pages = [
Page("testing.html", "Testing"),
Page("architecture.html", "Architecture"),
Page("sslkeylogfile.html", "TLS Master Secrets"),
# Page("addingviews.html", "Writing Content Views"),
# Page("addingviews.html", "Writing Content Views"),
]

8
doc-src/howmitmproxy.html
View File

@ -26,7 +26,7 @@ This is a proxy GET request - an extended form of the vanilla HTTP GET request
that includes a schema and host specification, and it includes all the
information mitmproxy needs to proceed.
<img src="explicit.png"/>
<img class="img-responsive" src="explicit.png"/>
<table class="table">
<tbody>
@ -158,7 +158,7 @@ handshake. Luckily, this is almost never an issue in practice.
Lets put all of this together into the complete explicitly proxied HTTPS flow.
<img src="explicit_https.png"/>
<img class="img-responsive" src="explicit_https.png"/>
<table class="table">
<tbody>
@ -250,7 +250,7 @@ mitmproxy, this takes the form of a built-in set of
that know how to talk to each platform's redirection mechanism. Once we have
this information, the process is fairly straight-forward.
<img src="transparent.png"/>
<img class="img-responsive" src="transparent.png"/>
<table class="table">
@ -296,7 +296,7 @@ transparently proxying HTTP, and explicitly proxying HTTPS. We use the routing
mechanism to establish the upstream server address, and then proceed as for
explicit HTTPS connections to establish the CN and SANs, and cope with SNI.
<img src="transparent_https.png"/>
<img class="img-responsive" src="transparent_https.png"/>
<table class="table">

11
doc-src/index.py
View File

@ -2,7 +2,7 @@ import os
import sys
import datetime
import countershape
from countershape import Page, Directory, markup, model
from countershape import Page, Directory, markup
import countershape.template
MITMPROXY_SRC = os.path.abspath(
@ -15,10 +15,10 @@ ns.VERSION = version.VERSION
if ns.options.website:
ns.idxpath = "doc/index.html"
this.layout = countershape.Layout("_websitelayout.html")
else:
ns.idxpath = "index.html"
this.layout = countershape.Layout("_layout.html")
this.layout = countershape.layout.FileLayout("_layout.html")
ns.title = countershape.template.Template(None, "<h1>@!this.title!@</h1>")
this.titlePrefix = "%s - " % version.NAMEVERSION
@ -37,7 +37,7 @@ def mpath(p):
def example(s):
d = file(mpath(s)).read().rstrip()
extemp = """<div class="example">%s<div class="example_legend">(%s)</div></div>"""
return extemp%(countershape.template.Syntax("py")(d), s)
return extemp % (countershape.template.Syntax("py")(d), s)
ns.example = example
@ -52,7 +52,8 @@ def nav(page, current, state):
else:
pre = "<li>"
p = state.application.getPage(page)
return pre + '<a href="%s">%s</a></li>'%(model.UrlTo(page), p.title)
return pre + \
'<a href="%s">%s</a></li>' % (countershape.widgets.UrlTo(page), p.title)
ns.nav = nav
ns.navbar = countershape.template.File(None, "_nav.html")

BIN
doc-src/mitmproxy-long.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 121 KiB

16
doc-src/mitmproxy.html
View File

@ -9,7 +9,7 @@ documentation from any __mitmproxy__ screen.
The flow list shows an index of captured flows in chronological order.
<img src="@!urlTo('screenshots/mitmproxy.png')!@"/>
<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy.png')!@"/>
- __1__: A GET request, returning a 302 Redirect response.
- __2__: A GET request, returning 16.75kb of text/html data.
@ -32,7 +32,7 @@ interfaces.
The __Flow View__ lets you inspect and manipulate a single flow:
<img src="@!urlTo('screenshots/mitmproxy-flowview.png')!@"/>
<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-flowview.png')!@"/>
- __1__: Flow summary.
- __2__: The Request/Response tabs, showing you which part of the flow you are
@ -65,13 +65,13 @@ At the moment, the Grid Editor is used in four parts of mitmproxy:
If there is is no data, an empty editor will be started to let you add some.
Here is the editor showing the headers from a request:
<img src="@!urlTo('screenshots/mitmproxy-kveditor.png')!@"/>
<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-kveditor.png')!@"/>
To edit, navigate to the key or value you want to modify using the arrow or vi
navigation keys, and press enter. The background color will change to show that
you are in edit mode for the specified field:
<img src="@!urlTo('screenshots/mitmproxy-kveditor-editmode.png')!@"/>
<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-kveditor-editmode.png')!@"/>
Modify the field as desired, then press escape to exit edit mode when you're
done. You can also add a row (_a_ key), delete a row (_d_ key), spawn an
@ -88,7 +88,7 @@ or client.
### 1: Set an interception pattern
<img src="@!urlTo('mitmproxy-intercept-filt.png')!@"/>
<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-filt.png')!@"/>
We press _i_ to set an interception pattern. In this case, the __~q__ filter
pattern tells __mitmproxy__ to intercept all requests. For complete filter
@ -97,18 +97,18 @@ document, or the built-in help function in __mitmproxy__.
### 2: Intercepted connections are indicated with orange text:
<img src="@!urlTo('mitmproxy-intercept-mid.png')!@"/>
<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-mid.png')!@"/>
### 3: You can now view and modify the request:
<img src="@!urlTo('mitmproxy-intercept-options.png')!@"/>
<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-options.png')!@"/>
In this case, we viewed the request by selecting it, pressed _e_ for "edit"
and _m_ for "method" to change the HTTP request method.
### 4: Accept the intercept to continue:
<img src="@!urlTo('mitmproxy-intercept-result.png')!@"/>
<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-result.png')!@"/>
Finally, we press _a_ to accept the modified request, which is then sent on to
the server. In this case, we changed the request from an HTTP GET to

16
doc-src/modes.html
View File

@ -9,7 +9,7 @@ variety of scenarios:
Now, which one should you pick? Use this flow chart:
<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
<div class="page-header">
<h1>Regular Proxy</h1>
@ -31,7 +31,7 @@ these cases, you need to use mitmproxy's transparent mode.
If you are proxying an external device, your network will probably look like this:
<img src="@!urlTo('schematics/proxy-modes-regular.png')!@">
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-regular.png')!@">
The square brackets signify the source and destination IP addresses. Your
client explicitly connects to mitmproxy and mitmproxy explicitly connects
@ -48,7 +48,7 @@ below, a machine running mitmproxy has been inserted between the router and
the internet:
<a href="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
<img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
</a>
The square brackets signify the source and destination IP addresses. Round
@ -60,7 +60,7 @@ remove the target information, leaving mitmproxy unable to determine the real
destination.
<a href="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@">
<img src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
<h2>Common Configurations</h2>
@ -79,7 +79,7 @@ intact, is to simply configure the client with the mitmproxy box as the
default gateway.
<a href="@!urlTo('schematics/proxy-modes-transparent-2.png')!@">
<img src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
In this scenario, we would:
@ -141,7 +141,7 @@ packet filter you're using. In most cases, the configuration will look like
this:
<a href="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
<img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
</a>
@ -154,7 +154,7 @@ Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal
HTTP server:
<a href="@!urlTo('schematics/proxy-modes-reverse.png')!@">
<img src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
</a>
There are various use-cases:
@ -215,7 +215,7 @@ appliance, you can use mitmproxy's upstream mode. In upstream mode, all
requests are unconditionally transferred to an upstream proxy of your choice.
<a href="@!urlTo('schematics/proxy-modes-upstream.png')!@">
<img src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy
mode. You could in theory chain multiple mitmproxy instances in a row, but

6
doc-src/tutorials/gamecenter.html
View File

@ -15,14 +15,14 @@ Worm](http://itunes.apple.com/us/app/super-mega-worm/id388541990?mt=8) - a
great little retro-apocalyptic sidescroller for the iPhone:
<center>
<img src="@!urlTo('tutorials/supermega.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/supermega.png')!@"/>
</center>
After finishing a game (take your time), watch the traffic flowing through
mitmproxy:
<center>
<img src="@!urlTo('tutorials/one.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/one.png')!@"/>
</center>
We see a bunch of things we might expect - initialisation, the retrieval of
@ -97,7 +97,7 @@ replay.
## The glorious result and some intrigue
<center>
<img src="@!urlTo('tutorials/leaderboard.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/leaderboard.png')!@"/>
</center>
And that's it - according to the records, I am the greatest Super Mega Worm

8
doc-src/tutorials/transparent-dhcp.html
View File

@ -4,10 +4,10 @@ This walkthrough illustrates how to set up transparent proxying with mitmproxy.
The network setup is simple: `internet <--> proxy vm <--> (virtual) internal network`.
For the proxy machine, *eth0* represents the outgoing network. *eth1* is connected to the internal network that will be proxified, using a static ip (192.168.3.1).
<hr>VirtualBox configuration:
<img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth0.png')!@"/><br><br>
<img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth1.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth0.png')!@"/><br><br>
<img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth1.png')!@"/>
<br>Proxy VM:
<img src="@!urlTo('tutorials/transparent-dhcp/step1_proxy.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_proxy.png')!@"/>
<hr>
2. **Configure DHCP and DNS**
We use dnsmasq to provide DHCP and DNS in our internal network.
@ -34,7 +34,7 @@ This walkthrough illustrates how to set up transparent proxying with mitmproxy.
`sudo service dnsmasq restart`
<hr>
Your proxied machine's network settings should now look similar to this:
<img src="@!urlTo('tutorials/transparent-dhcp/step2_proxied_vm.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step2_proxied_vm.png')!@"/>
<hr>
3. **Set up traffic redirection to mitmproxy**

14
examples/change_upstream_proxy.py
View File

@ -1,10 +1,13 @@
# This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy
# in upstream proxy mode.
#
# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s "change_upstream_proxy.py host"
# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s
# "change_upstream_proxy.py host"
from libmproxy.protocol.http import send_connect_request
alternative_upstream_proxy = ("localhost", 8082)
def should_redirect(flow):
return flow.request.host == "example.com"
@ -15,7 +18,12 @@ def request(context, flow):
# If you want to change the target server, you should modify flow.request.host and flow.request.port
# flow.live.change_server should only be used by inline scripts to change the upstream proxy,
# unless you are sure that you know what you are doing.
server_changed = flow.live.change_server(alternative_upstream_proxy, persistent_change=True)
server_changed = flow.live.change_server(
alternative_upstream_proxy,
persistent_change=True)
if flow.request.scheme == "https" and server_changed:
send_connect_request(flow.live.c.server_conn, flow.request.host, flow.request.port)
send_connect_request(
flow.live.c.server_conn,
flow.request.host,
flow.request.port)
flow.live.c.establish_ssl(server=True)

6
examples/dns_spoofing.py
View File

@ -25,9 +25,11 @@ mitmproxy -p 443 -R https2http://localhost:8000
def request(context, flow):
if flow.client_conn.ssl_established:
# TLS SNI or Host header
flow.request.host = flow.client_conn.connection.get_servername() or flow.request.pretty_host(hostheader=True)
flow.request.host = flow.client_conn.connection.get_servername(
) or flow.request.pretty_host(hostheader=True)
# If you use a https2http location as default destination, these attributes need to be corrected as well:
# If you use a https2http location as default destination, these
# attributes need to be corrected as well:
flow.request.port = 443
flow.request.scheme = "https"
else:

2
examples/filt.py
View File

@ -3,11 +3,13 @@
from libmproxy import filt
def start(context, argv):
if len(argv) != 2:
raise ValueError("Usage: -s 'filt.py FILTER'")
context.filter = filt.parse(argv[1])
def response(context, flow):
if flow.match(context.filter):
print("Flow matches filter:")

3
examples/flowbasic
View File

@ -36,7 +36,8 @@ class MyMaster(flow.FlowMaster):
config = proxy.ProxyConfig(
port=8080,
cadir="~/.mitmproxy/" # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
# use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
cadir="~/.mitmproxy/"
)
state = flow.State()
server = ProxyServer(config)

54
examples/har_extractor.py
View File

@ -83,7 +83,8 @@ def response(context, flow):
# Calculate the connect_time for this server_conn. Afterwards add it to
# seen list, in order to avoid the connect_time being present in entries
# that use an existing connection.
connect_time = flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start
connect_time = flow.server_conn.timestamp_tcp_setup - \
flow.server_conn.timestamp_start
context.seen_server.add(flow.server_conn)
if flow.server_conn.timestamp_ssl_setup is not None:
@ -91,7 +92,8 @@ def response(context, flow):
# the start of the successful tcp setup and the successful ssl
# setup. If no ssl setup has been made it is left as -1 since it
# doesn't apply to this connection.
ssl_time = flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup
ssl_time = flow.server_conn.timestamp_ssl_setup - \
flow.server_conn.timestamp_tcp_setup
# Calculate the raw timings from the different timestamps present in the
# request and response object. For lack of a way to measure it dns timings
@ -110,7 +112,8 @@ def response(context, flow):
# HAR timings are integers in ms, so we have to re-encode the raw timings to
# that format.
timings = dict([(key, int(1000 * value)) for key, value in timings_raw.iteritems()])
timings = dict([(key, int(1000 * value))
for key, value in timings_raw.iteritems()])
# The full_time is the sum of all timings. Timings set to -1 will be ignored
# as per spec.
@ -119,20 +122,27 @@ def response(context, flow):
if item > -1:
full_time += item
started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, tz=utc).isoformat()
started_date_time = datetime.fromtimestamp(
flow.request.timestamp_start,
tz=utc).isoformat()
request_query_string = [{"name": k, "value": v} for k, v in flow.request.get_query()]
request_query_string = [{"name": k, "value": v}
for k, v in flow.request.get_query()]
request_http_version = ".".join([str(v) for v in flow.request.httpversion])
# Cookies are shaped as tuples by MITMProxy.
request_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.request.get_cookies() or {}).iteritems()]
request_cookies = [{"name": k.strip(), "value": v[0]}
for k, v in (flow.request.get_cookies() or {}).iteritems()]
request_headers = [{"name": k, "value": v} for k, v in flow.request.headers]
request_headers_size = len(str(flow.request.headers))
request_body_size = len(flow.request.content)
response_http_version = ".".join([str(v) for v in flow.response.httpversion])
response_http_version = ".".join(
[str(v) for v in flow.response.httpversion])
# Cookies are shaped as tuples by MITMProxy.
response_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.response.get_cookies() or {}).iteritems()]
response_headers = [{"name": k, "value": v} for k, v in flow.response.headers]
response_cookies = [{"name": k.strip(), "value": v[0]}
for k, v in (flow.response.get_cookies() or {}).iteritems()]
response_headers = [{"name": k, "value": v}
for k, v in flow.response.headers]
response_headers_size = len(str(flow.response.headers))
response_body_size = len(flow.response.content)
response_body_decoded_size = len(flow.response.get_decoded_content())
@ -140,33 +150,43 @@ def response(context, flow):
response_mime_type = flow.response.headers.get_first('Content-Type', '')
response_redirect_url = flow.response.headers.get_first('Location', '')
entry = HAR.entries({"startedDateTime": started_date_time,
entry = HAR.entries(
{
"startedDateTime": started_date_time,
"time": full_time,
"request": {"method": flow.request.method,
"request": {
"method": flow.request.method,
"url": flow.request.url,
"httpVersion": request_http_version,
"cookies": request_cookies,
"headers": request_headers,
"queryString": request_query_string,
"headersSize": request_headers_size,
"bodySize": request_body_size, },
"response": {"status": flow.response.code,
"bodySize": request_body_size,
},
"response": {
"status": flow.response.code,
"statusText": flow.response.msg,
"httpVersion": response_http_version,
"cookies": response_cookies,
"headers": response_headers,
"content": {"size": response_body_size,
"content": {
"size": response_body_size,
"compression": response_body_compression,
"mimeType": response_mime_type},
"redirectURL": response_redirect_url,
"headersSize": response_headers_size,
"bodySize": response_body_size, },
"bodySize": response_body_size,
},
"cache": {},
"timings": timings, })
"timings": timings,
})
# If the current url is in the page list of context.HARLog or does not have
# a referrer we add it as a new pages object.
if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get('Referer', None) is None:
if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get(
'Referer',
None) is None:
page_id = context.HARLog.create_page_id()
context.HARLog.add(
HAR.pages({

7
examples/iframe_injector.py
View File

@ -16,7 +16,12 @@ def response(context, flow):
with decoded(flow.response): # Remove content encoding (gzip, ...)
html = BeautifulSoup(flow.response.content)
if html.body:
iframe = html.new_tag("iframe", src=context.iframe_url, frameborder=0, height=0, width=0)
iframe = html.new_tag(
"iframe",
src=context.iframe_url,
frameborder=0,
height=0,
width=0)
html.body.insert(0, iframe)
flow.response.content = str(html)
context.log("Iframe inserted.")

1
examples/ignore_websocket.py
View File

@ -24,6 +24,7 @@ def done(context):
HTTPRequest._headers_to_strip_off.append("Connection")
HTTPRequest._headers_to_strip_off.append("Upgrade")
@concurrent
def response(context, flow):
value = flow.response.headers.get_first("Connection", None)

52
examples/mitmproxywrapper.py
View File

@ -14,6 +14,7 @@ import contextlib
import os
import sys
class Wrapper(object):
def __init__(self, port, extra_arguments=None):
@ -21,16 +22,21 @@ class Wrapper(object):
self.extra_arguments = extra_arguments
def run_networksetup_command(self, *arguments):
return subprocess.check_output(['sudo', 'networksetup'] + list(arguments))
return subprocess.check_output(
['sudo', 'networksetup'] + list(arguments))
def proxy_state_for_service(self, service):
state = self.run_networksetup_command('-getwebproxy', service).splitlines()
state = self.run_networksetup_command(
'-getwebproxy',
service).splitlines()
return dict([re.findall(r'([^:]+): (.*)', line)[0] for line in state])
def enable_proxy_for_service(self, service):
print 'Enabling proxy on {}...'.format(service)
for subcommand in ['-setwebproxy', '-setsecurewebproxy']:
self.run_networksetup_command(subcommand, service, '127.0.0.1', str(self.port))
self.run_networksetup_command(
subcommand, service, '127.0.0.1', str(
self.port))
def disable_proxy_for_service(self, service):
print 'Disabling proxy on {}...'.format(service)
@ -39,11 +45,17 @@ class Wrapper(object):
def interface_name_to_service_name_map(self):
order = self.run_networksetup_command('-listnetworkserviceorder')
mapping = re.findall(r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$', order, re.MULTILINE)
mapping = re.findall(
r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$',
order,
re.MULTILINE)
return dict([(b, a) for (a, b) in mapping])
def run_command_with_input(self, command, input):
popen = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
popen = subprocess.Popen(
command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
(stdout, stderr) = popen.communicate(input)
return stdout
@ -54,13 +66,15 @@ class Wrapper(object):
return interface
def primary_service_name(self):
return self.interface_name_to_service_name_map()[self.primary_interace_name()]
return self.interface_name_to_service_name_map()[
self.primary_interace_name()]
def proxy_enabled_for_service(self, service):
return self.proxy_state_for_service(service)['Enabled'] == 'Yes'
def toggle_proxy(self):
new_state = not self.proxy_enabled_for_service(self.primary_service_name())
new_state = not self.proxy_enabled_for_service(
self.primary_service_name())
for service_name in self.connected_service_names():
if self.proxy_enabled_for_service(service_name) and not new_state:
self.disable_proxy_for_service(service_name)
@ -74,8 +88,11 @@ class Wrapper(object):
service_names = []
for service_id in service_ids:
scutil_script = 'show Setup:/Network/Service/{}\n'.format(service_id)
stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
scutil_script = 'show Setup:/Network/Service/{}\n'.format(
service_id)
stdout = self.run_command_with_input(
'/usr/sbin/scutil',
scutil_script)
service_name, = re.findall(r'UserDefinedName\s*:\s*(.+)', stdout)
service_names.append(service_name)
@ -119,11 +136,19 @@ class Wrapper(object):
def main(cls):
parser = argparse.ArgumentParser(
description='Helper tool for OS X proxy configuration and mitmproxy.',
epilog='Any additional arguments will be passed on unchanged to mitmproxy.'
)
parser.add_argument('-t', '--toggle', action='store_true', help='just toggle the proxy configuration')
epilog='Any additional arguments will be passed on unchanged to mitmproxy.')
parser.add_argument(
'-t',
'--toggle',
action='store_true',
help='just toggle the proxy configuration')
# parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy')
parser.add_argument('-p', '--port', type=int, help='override the default port of 8080', default=8080)
parser.add_argument(
'-p',
'--port',
type=int,
help='override the default port of 8080',
default=8080)
args, extra_arguments = parser.parse_known_args()
wrapper = cls(port=args.port, extra_arguments=extra_arguments)
@ -139,4 +164,3 @@ class Wrapper(object):
if __name__ == '__main__':
Wrapper.ensure_superuser()
Wrapper.main()

3
examples/modify_form.py
View File

@ -1,6 +1,7 @@
def request(context, flow):
if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]:
if "application/x-www-form-urlencoded" in flow.request.headers[
"content-type"]:
form = flow.request.get_form_urlencoded()
form["mitmproxy"] = ["rocks"]
flow.request.set_form_urlencoded(form)

7
examples/modify_response_body.py
View File

@ -6,10 +6,13 @@ from libmproxy.protocol.http import decoded
def start(context, argv):
if len(argv) != 3:
raise ValueError('Usage: -s "modify-response-body.py old new"')
# You may want to use Python's argparse for more sophisticated argument parsing.
# You may want to use Python's argparse for more sophisticated argument
# parsing.
context.old, context.new = argv[1], argv[2]
def response(context, flow):
with decoded(flow.response): # automatically decode gzipped responses.
flow.response.content = flow.response.content.replace(context.old, context.new)
flow.response.content = flow.response.content.replace(
context.old,
context.new)

5
examples/read_dumpfile
View File

@ -4,7 +4,8 @@
#
from libmproxy import flow
import json, sys
import json
import sys
with open("logfile", "rb") as logfile:
freader = flow.FlowReader(logfile)
@ -14,5 +15,5 @@ with open("logfile", "rb") as logfile:
print(f.request.host)
json.dump(f.get_state(), sys.stdout, indent=4)
print ""
except flow.FlowReadError, v:
except flow.FlowReadError as v:
print "Flow file corrupted. Stopped loading."

3
examples/redirect_requests.py
View File

@ -8,7 +8,8 @@ This example shows two ways to redirect flows to other destinations.
def request(context, flow):
# pretty_host(hostheader=True) takes the Host: header of the request into account,
# which is useful in transparent mode where we usually only have the IP otherwise.
# which is useful in transparent mode where we usually only have the IP
# otherwise.
# Method 1: Answer with a locally generated response
if flow.request.pretty_host(hostheader=True).endswith("example.com"):

9
examples/stub.py
View File

@ -1,12 +1,15 @@
"""
This is a script stub, with definitions for all events.
"""
def start(context, argv):
"""
Called once on script startup, before any other events.
"""
context.log("start")
def clientconnect(context, conn_handler):
"""
Called when a client initiates a connection to the proxy. Note that a
@ -14,6 +17,7 @@ def clientconnect(context, conn_handler):
"""
context.log("clientconnect")
def serverconnect(context, conn_handler):
"""
Called when the proxy initiates a connection to the target server. Note that a
@ -21,6 +25,7 @@ def serverconnect(context, conn_handler):
"""
context.log("serverconnect")
def request(context, flow):
"""
Called when a client request has been received.
@ -36,12 +41,14 @@ def responseheaders(context, flow):
"""
context.log("responseheaders")
def response(context, flow):
"""
Called when a server response has been received.
"""
context.log("response")
def error(context, flow):
"""
Called when a flow error has occured, e.g. invalid server responses, or
@ -50,12 +57,14 @@ def error(context, flow):
"""
context.log("error")
def clientdisconnect(context, conn_handler):
"""
Called when a client disconnects from the proxy.
"""
context.log("clientdisconnect")
def done(context):
"""
Called once on script shutdown, after any other events.

1
examples/upsidedownternet.py
View File

@ -2,6 +2,7 @@ import cStringIO
from PIL import Image
from libmproxy.protocol.http import decoded
def response(context, flow):
if flow.response.headers.get_first("content-type", "").startswith("image"):
with decoded(flow.response): # automatically decode gzipped responses.

20
libmproxy/cmdline.py
View File

@ -65,7 +65,7 @@ def parse_replace_hook(s):
patt, regex, replacement = _parse_hook(s)
try:
re.compile(regex)
except re.error, e:
except re.error as e:
raise ParseException("Malformed replacement regex: %s" % str(e.message))
return patt, regex, replacement
@ -127,7 +127,6 @@ def parse_server_spec_special(url):
return ret
def get_common_options(options):
stickycookie, stickyauth = None, None
if options.stickycookie_filt:
@ -142,17 +141,17 @@ def get_common_options(options):
for i in options.replace:
try:
p = parse_replace_hook(i)
except ParseException, e:
except ParseException as e:
raise configargparse.ArgumentTypeError(e.message)
reps.append(p)
for i in options.replace_file:
try:
patt, rex, path = parse_replace_hook(i)
except ParseException, e:
except ParseException as e:
raise configargparse.ArgumentTypeError(e.message)
try:
v = open(path, "rb").read()
except IOError, e:
except IOError as e:
raise configargparse.ArgumentTypeError(
"Could not read replace file: %s" % path
)
@ -162,7 +161,7 @@ def get_common_options(options):
for i in options.setheader:
try:
p = parse_setheader(i)
except ParseException, e:
except ParseException as e:
raise configargparse.ArgumentTypeError(e.message)
setheaders.append(p)
@ -221,7 +220,7 @@ def common_options(parser):
parser.add_argument(
"--cadir",
action="store", type=str, dest="cadir", default=config.CA_DIR,
help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR
help="Location of the default mitmproxy CA files. (%s)" % config.CA_DIR
)
parser.add_argument(
"--host",
@ -482,9 +481,10 @@ def common_options(parser):
)
group.add_argument(
"--replay-ignore-host",
action="store_true", dest="replay_ignore_host", default=False,
help="Ignore request's destination host while searching for a saved flow to replay"
)
action="store_true",
dest="replay_ignore_host",
default=False,
help="Ignore request's destination host while searching for a saved flow to replay")
group = parser.add_argument_group(
"Replacements",

29
libmproxy/console/__init__.py
View File

@ -85,10 +85,10 @@ class ConsoleState(flow.State):
return self.view[pos], pos
def get_next(self, pos):
return self.get_from_pos(pos+1)
return self.get_from_pos(pos + 1)
def get_prev(self, pos):
return self.get_from_pos(pos-1)
return self.get_from_pos(pos - 1)
def delete_flow(self, f):
if f in self.view and self.view.index(f) <= self.focus:
@ -255,7 +255,7 @@ class ConsoleMaster(flow.FlowMaster):
try:
f = file(path, mode)
self.start_stream(f, None)
except IOError, v:
except IOError as v:
return str(v)
self.stream_path = path
@ -263,22 +263,24 @@ class ConsoleMaster(flow.FlowMaster):
status, val = s.run(method, f)
if val:
if status:
self.add_event("Method %s return: %s"%(method, val), "debug")
self.add_event("Method %s return: %s" % (method, val), "debug")
else:
self.add_event("Method %s error: %s"%(method, val[1]), "error")
self.add_event(
"Method %s error: %s" %
(method, val[1]), "error")
def run_script_once(self, command, f):
if not command:
return
self.add_event("Running script on flow: %s"%command, "debug")
self.add_event("Running script on flow: %s" % command, "debug")
try:
s = script.Script(command, self)
except script.ScriptError, v:
except script.ScriptError as v:
signals.status_message.send(
message = "Error loading script."
)
self.add_event("Error loading script:\n%s"%v.args[0], "error")
self.add_event("Error loading script:\n%s" % v.args[0], "error")
return
if f.request:
@ -562,7 +564,7 @@ class ConsoleMaster(flow.FlowMaster):
for i in flows:
fw.add(i)
f.close()
except IOError, v:
except IOError as v:
signals.status_message.send(message=v.strerror)
def save_one_flow(self, path, flow):
@ -575,13 +577,13 @@ class ConsoleMaster(flow.FlowMaster):
if not path:
return
ret = self.load_flows_path(path)
return ret or "Flows loaded from %s"%path
return ret or "Flows loaded from %s" % path
def load_flows_path(self, path):
reterr = None
try:
flow.FlowMaster.load_flows_file(self, path)
except flow.FlowReadError, v:
except flow.FlowReadError as v:
reterr = str(v)
signals.flowlist_change.send(self)
return reterr
@ -652,7 +654,8 @@ class ConsoleMaster(flow.FlowMaster):
)
def process_flow(self, f):
if self.state.intercept and f.match(self.state.intercept) and not f.request.is_replay:
if self.state.intercept and f.match(
self.state.intercept) and not f.request.is_replay:
f.intercept(self)
else:
f.reply()
@ -674,7 +677,7 @@ class ConsoleMaster(flow.FlowMaster):
self.eventlist.append(e)
if len(self.eventlist) > EVENTLOG_SIZE:
self.eventlist.pop(0)
self.eventlist.set_focus(len(self.eventlist)-1)
self.eventlist.set_focus(len(self.eventlist) - 1)
# Handlers
def handle_error(self, f):

6
libmproxy/console/common.py
View File

@ -164,7 +164,7 @@ def raw_format_flow(f, focus, extended, padding):
4: "code_400",
5: "code_500",
}
ccol = codes.get(f["resp_code"]/100, "code_other")
ccol = codes.get(f["resp_code"] / 100, "code_other")
resp.append(fcol(SYMBOL_RETURN, ccol))
if f["resp_is_replay"]:
resp.append(fcol(SYMBOL_REPLAY, "replay"))
@ -200,7 +200,7 @@ def save_data(path, data, master, state):
try:
with file(path, "wb") as f:
f.write(data)
except IOError, v:
except IOError as v:
signals.status_message.send(message=v.strerror)
@ -214,7 +214,7 @@ def ask_save_overwite(path, data, master, state):
save_data(path, data, master, state)
signals.status_prompt_onekey.send(
prompt = "'"+path+"' already exists. Overwite?",
prompt = "'" + path + "' already exists. Overwite?",
keys = (
("yes", "y"),
("no", "n"),

19
libmproxy/console/contentview.py
View File

@ -36,7 +36,7 @@ else:
cssutils.ser.prefs.indentClosingBrace = False
cssutils.ser.prefs.validOnly = False
VIEW_CUTOFF = 1024*50
VIEW_CUTOFF = 1024 * 50
def _view_text(content, total, limit):
@ -59,7 +59,7 @@ def trailer(clen, txt, limit):
txt.append(
urwid.Text(
[
("highlight", "... %s of data not shown. Press "%netlib.utils.pretty_size(rem)),
("highlight", "... %s of data not shown. Press " % netlib.utils.pretty_size(rem)),
("key", "f"),
("highlight", " to load all data.")
]
@ -76,7 +76,7 @@ class ViewAuto:
ctype = hdrs.get_first("content-type")
if ctype:
ct = utils.parse_content_type(ctype) if ctype else None
ct = "%s/%s"%(ct[0], ct[1])
ct = "%s/%s" % (ct[0], ct[1])
if ct in content_types_map:
return content_types_map[ct][0](hdrs, content, limit)
elif utils.isXML(content):
@ -227,7 +227,7 @@ class ViewURLEncoded:
lines = utils.urldecode(content)
if lines:
body = common.format_keyvals(
[(k+":", v) for (k, v) in lines],
[(k + ":", v) for (k, v) in lines],
key = "header",
val = "text"
)
@ -304,7 +304,6 @@ if pyamf:
if not envelope:
return None
txt = []
for target, message in iter(envelope):
if isinstance(message, pyamf.remoting.Request):
@ -315,13 +314,13 @@ if pyamf:
else:
txt.append(urwid.Text([
("header", "Response: "),
("text", "%s, code %s"%(target, message.status)),
("text", "%s, code %s" % (target, message.status)),
]))
s = json.dumps(self.unpack(message), indent=4)
txt.extend(_view_text(s[:limit], len(s), limit))
return "AMF v%s"%envelope.amfVersion, txt
return "AMF v%s" % envelope.amfVersion, txt
class ViewJavaScript:
@ -375,7 +374,7 @@ class ViewImage:
return None
parts = [
("Format", str(img.format_description)),
("Size", "%s x %s px"%img.size),
("Size", "%s x %s px" % img.size),
("Mode", str(img.mode)),
]
for i in sorted(img.info.keys()):
@ -401,7 +400,7 @@ class ViewImage:
key = "header",
val = "text"
)
return "%s image"%img.format, fmt
return "%s image" % img.format, fmt
class ViewProtobuf:
@ -526,7 +525,7 @@ def get_content_view(viewmode, hdrItems, content, limit, logfunc, is_request):
decoded = encoding.decode(enc, content)
if decoded:
content = decoded
msg.append("[decoded %s]"%enc)
msg.append("[decoded %s]" % enc)
try:
ret = viewmode(hdrs, content, limit)
# Third-party viewers can fail in unexpected ways...

2
libmproxy/console/flowdetailview.py
View File

@ -34,7 +34,7 @@ def flowdetails(state, flow):
if c:
text.append(urwid.Text([("head", "Server Certificate:")]))
parts = [
["Type", "%s, %s bits"%c.keyinfo],
["Type", "%s, %s bits" % c.keyinfo],
["SHA1 digest", c.digest("sha1")],
["Valid to", str(c.notafter)],
["Valid from", str(c.notbefore)],

8
libmproxy/console/flowlist.py
View File

@ -50,7 +50,7 @@ class EventListBox(urwid.ListBox):
elif key == "G":
self.set_focus(0)
elif key == "g":
self.set_focus(len(self.master.eventlist)-1)
self.set_focus(len(self.master.eventlist) - 1)
return urwid.ListBox.keypress(self, size, key)
@ -76,7 +76,8 @@ class BodyPile(urwid.Pile):
def keypress(self, size, key):
if key == "tab":
self.focus_position = (self.focus_position + 1)%len(self.widget_list)
self.focus_position = (
self.focus_position + 1) % len(self.widget_list)
if self.focus_position == 1:
self.widget_list[1].header = self.active_header
else:
@ -157,7 +158,8 @@ class ConnectionItem(urwid.WidgetWrap):
callback = self.master.server_playback_path
)
def keypress(self, (maxcol,), key):
def keypress(self, xxx_todo_changeme, key):
(maxcol,) = xxx_todo_changeme
key = common.shortcuts(key)
if key == "a":
self.flow.accept_intercept(self.master)

15
libmproxy/console/flowview.py
View File

@ -173,7 +173,7 @@ class FlowView(tabs.Tabs):
False
)
if full:
limit = sys.maxint
limit = sys.maxsize
else:
limit = contentview.VIEW_CUTOFF
description, text_objects = cache.get(
@ -197,7 +197,7 @@ class FlowView(tabs.Tabs):
def conn_text(self, conn):
if conn:
txt = common.format_keyvals(
[(h+":", v) for (h, v) in conn.headers.lst],
[(h + ":", v) for (h, v) in conn.headers.lst],
key = "header",
val = "text"
)
@ -217,7 +217,7 @@ class FlowView(tabs.Tabs):
" ",
('heading', "["),
('heading_key', "m"),
('heading', (":%s]"%viewmode.name)),
('heading', (":%s]" % viewmode.name)),
],
align="right"
)
@ -272,8 +272,9 @@ class FlowView(tabs.Tabs):
except ValueError:
return None
import BaseHTTPServer
if BaseHTTPServer.BaseHTTPRequestHandler.responses.has_key(int(code)):
response.msg = BaseHTTPServer.BaseHTTPRequestHandler.responses[int(code)][0]
if int(code) in BaseHTTPServer.BaseHTTPRequestHandler.responses:
response.msg = BaseHTTPServer.BaseHTTPRequestHandler.responses[
int(code)][0]
signals.flow_change.send(self, flow = self.flow)
def set_resp_msg(self, msg):
@ -494,7 +495,7 @@ class FlowView(tabs.Tabs):
elif key == "d":
if self.state.flow_count() == 1:
self.master.view_flowlist()
elif self.state.view.index(self.flow) == len(self.state.view)-1:
elif self.state.view.index(self.flow) == len(self.state.view) - 1:
self.view_prev_flow(self.flow)
else:
self.view_next_flow(self.flow)
@ -615,7 +616,7 @@ class FlowView(tabs.Tabs):
if conn.content:
t = conn.headers["content-type"] or [None]
t = t[0]
if os.environ.has_key("EDITOR") or os.environ.has_key("PAGER"):
if "EDITOR" in os.environ or "PAGER" in os.environ:
self.master.spawn_external_viewer(conn.content, t)
else:
signals.status_message.send(

23
libmproxy/console/grideditor.py
View File

@ -175,6 +175,7 @@ class GridWalker(urwid.ListWalker):
and errors is a set with an entry of each offset in rows that is an
error.
"""
def __init__(self, lst, editor):
self.lst = [(i, set([])) for i in lst]
self.editor = editor
@ -225,7 +226,7 @@ class GridWalker(urwid.ListWalker):
def delete_focus(self):
if self.lst:
del self.lst[self.focus]
self.focus = min(len(self.lst)-1, self.focus)
self.focus = min(len(self.lst) - 1, self.focus)
self._modified()
def _insert(self, pos):
@ -266,14 +267,14 @@ class GridWalker(urwid.ListWalker):
self._modified()
def right(self):
self.focus_col = min(self.focus_col + 1, len(self.editor.columns)-1)
self.focus_col = min(self.focus_col + 1, len(self.editor.columns) - 1)
self._modified()
def tab_next(self):
self.stop_edit()
if self.focus_col < len(self.editor.columns)-1:
if self.focus_col < len(self.editor.columns) - 1:
self.focus_col += 1
elif self.focus != len(self.lst)-1:
elif self.focus != len(self.lst) - 1:
self.focus_col = 0
self.focus += 1
self._modified()
@ -297,14 +298,14 @@ class GridWalker(urwid.ListWalker):
self._modified()
def get_next(self, pos):
if pos+1 >= len(self.lst):
if pos + 1 >= len(self.lst):
return None, None
return GridRow(None, False, self.editor, self.lst[pos+1]), pos+1
return GridRow(None, False, self.editor, self.lst[pos + 1]), pos + 1
def get_prev(self, pos):
if pos-1 < 0:
if pos - 1 < 0:
return None, None
return GridRow(None, False, self.editor, self.lst[pos-1]), pos-1
return GridRow(None, False, self.editor, self.lst[pos - 1]), pos - 1
class GridListBox(urwid.ListBox):
@ -387,7 +388,7 @@ class GridEditor(urwid.WidgetWrap):
d = file(p, "rb").read()
self.walker.set_current_value(d, unescaped)
self.walker._modified()
except IOError, v:
except IOError as v:
return str(v)
def set_subeditor_value(self, val, focus, focus_col):
@ -418,7 +419,7 @@ class GridEditor(urwid.WidgetWrap):
elif key == "G":
self.walker.set_focus(0)
elif key == "g":
self.walker.set_focus(len(self.walker.lst)-1)
self.walker.set_focus(len(self.walker.lst) - 1)
elif key in ["h", "left"]:
self.walker.left()
elif key in ["l", "right"]:
@ -633,7 +634,7 @@ class ScriptEditor(GridEditor):
def is_error(self, col, val):
try:
script.Script.parse_command(val)
except script.ScriptError, v:
except script.ScriptError as v:
return str(v)

15
libmproxy/console/help.py
View File

@ -6,7 +6,7 @@ from . import common, signals
from .. import filt, version
footer = [
("heading", 'mitmproxy v%s '%version.VERSION),
("heading", 'mitmproxy v%s ' % version.VERSION),
('heading_key', "q"), ":back ",
]
@ -33,7 +33,12 @@ class HelpView(urwid.ListBox):
("pg up/down", "page up/down"),
("arrows", "up, down, left, right"),
]
text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
text.extend(
common.format_keyvals(
keys,
key="key",
val="text",
indent=4))
text.append(urwid.Text([("head", "\n\nGlobal keys:\n")]))
keys = [
@ -52,15 +57,15 @@ class HelpView(urwid.ListBox):
f = []
for i in filt.filt_unary:
f.append(
("~%s"%i.code, i.help)
("~%s" % i.code, i.help)
)
for i in filt.filt_rex:
f.append(
("~%s regex"%i.code, i.help)
("~%s regex" % i.code, i.help)
)
for i in filt.filt_int:
f.append(
("~%s int"%i.code, i.help)
("~%s int" % i.code, i.help)
)
f.sort()
f.extend(

1
libmproxy/console/options.py
View File

@ -8,6 +8,7 @@ footer = [
('heading_key', "C"), ":clear all ",
]
def _mkhelp():
text = []
keys = [

2
libmproxy/console/palettes.py
View File

@ -270,7 +270,7 @@ class SolarizedDark(LowDark):
# Status bar & heading
heading = (sol_base2, sol_base01),
heading_key = (sol_blue+",bold", sol_base01),
heading_key = (sol_blue + ",bold", sol_base01),
heading_inactive = (sol_base1, sol_base02),
# Help

2
libmproxy/console/pathedit.py
View File

@ -32,7 +32,7 @@ class _PathCompleter:
files = glob.glob(os.path.join(path, "*"))
prefix = txt
else:
files = glob.glob(path+"*")
files = glob.glob(path + "*")
prefix = os.path.dirname(txt)
prefix = prefix or "./"
for f in files:

6
libmproxy/console/searchable.py
View File

@ -37,7 +37,7 @@ class Searchable(urwid.ListBox):
self.set_focus(0)
self.walker._modified()
elif key == "g":
self.set_focus(len(self.walker)-1)
self.set_focus(len(self.walker) - 1)
self.walker._modified()
else:
return super(self.__class__, self).keypress(size, key)
@ -74,11 +74,11 @@ class Searchable(urwid.ListBox):
return
# Start search at focus + 1
if backwards:
rng = xrange(len(self.body)-1, -1, -1)
rng = xrange(len(self.body) - 1, -1, -1)
else:
rng = xrange(1, len(self.body) + 1)
for i in rng:
off = (self.focus_position + i)%len(self.body)
off = (self.focus_position + i) % len(self.body)
w = self.body[off]
txt = self.get_text(w)
if txt and self.search_term in txt:

18
libmproxy/console/select.py
View File

@ -2,6 +2,7 @@ import urwid
from . import common
class _OptionWidget(urwid.WidgetWrap):
def __init__(self, option, text, shortcut, active, focus):
self.option = option
@ -47,14 +48,14 @@ class OptionWalker(urwid.ListWalker):
return self.options[self.focus].render(True), self.focus
def get_next(self, pos):
if pos >= len(self.options)-1:
if pos >= len(self.options) - 1:
return None, None
return self.options[pos+1].render(False), pos+1
return self.options[pos + 1].render(False), pos + 1
def get_prev(self, pos):
if pos <= 0:
return None, None
return self.options[pos-1].render(False), pos-1
return self.options[pos - 1].render(False), pos - 1
class Heading:
@ -69,6 +70,8 @@ class Heading:
_neg = lambda: False
class Option:
def __init__(self, text, shortcut, getstate=None, activate=None):
self.text = text
@ -77,7 +80,12 @@ class Option:
self.activate = activate or _neg
def render(self, focus):
return _OptionWidget(self, self.text, self.shortcut, self.getstate(), focus)
return _OptionWidget(
self,
self.text,
self.shortcut,
self.getstate(),
focus)
class Select(urwid.ListBox):
@ -92,7 +100,7 @@ class Select(urwid.ListBox):
for i in options:
if hasattr(i, "shortcut") and i.shortcut:
if i.shortcut in self.keymap:
raise ValueError("Duplicate shortcut key: %s"%i.shortcut)
raise ValueError("Duplicate shortcut key: %s" % i.shortcut)
self.keymap[i.shortcut] = i
def keypress(self, size, key):

30
libmproxy/console/statusbar.py
View File

@ -58,7 +58,7 @@ class ActionBar(urwid.WidgetWrap):
mkup = []
for i, e in enumerate(keys):
mkup.extend(common.highlight_key(e[0], e[1]))
if i < len(keys)-1:
if i < len(keys) - 1:
mkup.append(",")
prompt.extend(mkup)
prompt.append(")? ")
@ -136,14 +136,14 @@ class StatusBar(urwid.WidgetWrap):
if self.master.client_playback:
r.append("[")
r.append(("heading_key", "cplayback"))
r.append(":%s to go]"%self.master.client_playback.count())
r.append(":%s to go]" % self.master.client_playback.count())
if self.master.server_playback:
r.append("[")
r.append(("heading_key", "splayback"))
if self.master.nopop:
r.append(":%s in file]"%self.master.server_playback.count())
r.append(":%s in file]" % self.master.server_playback.count())
else:
r.append(":%s to go]"%self.master.server_playback.count())
r.append(":%s to go]" % self.master.server_playback.count())
if self.master.get_ignore_filter():
r.append("[")
r.append(("heading_key", "I"))
@ -155,23 +155,23 @@ class StatusBar(urwid.WidgetWrap):
if self.master.state.intercept_txt:
r.append("[")
r.append(("heading_key", "i"))
r.append(":%s]"%self.master.state.intercept_txt)
r.append(":%s]" % self.master.state.intercept_txt)
if self.master.state.limit_txt:
r.append("[")
r.append(("heading_key", "l"))
r.append(":%s]"%self.master.state.limit_txt)
r.append(":%s]" % self.master.state.limit_txt)
if self.master.stickycookie_txt:
r.append("[")
r.append(("heading_key", "t"))
r.append(":%s]"%self.master.stickycookie_txt)
r.append(":%s]" % self.master.stickycookie_txt)
if self.master.stickyauth_txt:
r.append("[")
r.append(("heading_key", "u"))
r.append(":%s]"%self.master.stickyauth_txt)
r.append(":%s]" % self.master.stickyauth_txt)
if self.master.state.default_body_view.name != "Auto":
r.append("[")
r.append(("heading_key", "M"))
r.append(":%s]"%self.master.state.default_body_view.name)
r.append(":%s]" % self.master.state.default_body_view.name)
opts = []
if self.master.anticache:
@ -196,22 +196,22 @@ class StatusBar(urwid.WidgetWrap):
)
if opts:
r.append("[%s]"%(":".join(opts)))
r.append("[%s]" % (":".join(opts)))
if self.master.server.config.mode in ["reverse", "upstream"]:
dst = self.master.server.config.mode.dst
scheme = "https" if dst[0] else "http"
if dst[1] != dst[0]:
scheme += "2https" if dst[1] else "http"
r.append("[dest:%s]"%utils.unparse_url(scheme, *dst[2:]))
r.append("[dest:%s]" % utils.unparse_url(scheme, *dst[2:]))
if self.master.scripts:
r.append("[")
r.append(("heading_key", "s"))
r.append("cripts:%s]"%len(self.master.scripts))
r.append("cripts:%s]" % len(self.master.scripts))
# r.append("[lt:%0.3f]"%self.master.looptime)
if self.master.stream:
r.append("[W:%s]"%self.master.stream_path)
r.append("[W:%s]" % self.master.stream_path)
return r
@ -222,14 +222,14 @@ class StatusBar(urwid.WidgetWrap):
else:
offset = min(self.master.state.focus + 1, fc)
t = [
('heading', ("[%s/%s]"%(offset, fc)).ljust(9))
('heading', ("[%s/%s]" % (offset, fc)).ljust(9))
]
if self.master.server.bound:
host = self.master.server.address.host
if host == "0.0.0.0":
host = "*"
boundaddr = "[%s:%s]"%(host, self.master.server.address.port)
boundaddr = "[%s:%s]" % (host, self.master.server.address.port)
else:
boundaddr = ""
t.extend(self.get_status())

5
libmproxy/console/tabs.py
View File

@ -1,5 +1,6 @@
import urwid
class Tabs(urwid.WidgetWrap):
def __init__(self, tabs, tab_offset=0):
urwid.WidgetWrap.__init__(self, "")
@ -15,10 +16,10 @@ class Tabs(urwid.WidgetWrap):
def keypress(self, size, key):
if key in ["tab", "l"]:
self.tab_offset = (self.tab_offset + 1)%(len(self.tabs))
self.tab_offset = (self.tab_offset + 1) % (len(self.tabs))
self.show()
elif key == "h":
self.tab_offset = (self.tab_offset - 1)%(len(self.tabs))
self.tab_offset = (self.tab_offset - 1) % (len(self.tabs))
self.show()
return self._w.keypress(size, key)

11
libmproxy/controller.py
View File

@ -1,11 +1,14 @@
from __future__ import absolute_import
import Queue, threading
import Queue
import threading
class DummyReply:
"""
A reply object that does nothing. Useful when we need an object to seem
like it has a channel, and during testing.
"""
def __init__(self):
self.acked = False
@ -19,6 +22,7 @@ class Reply:
This object is used to respond to the message through the return
channel.
"""
def __init__(self, obj):
self.obj = obj
self.q = Queue.Queue()
@ -67,11 +71,13 @@ class Slave(threading.Thread):
Slaves get a channel end-point through which they can send messages to
the master.
"""
def __init__(self, channel, server):
self.channel, self.server = channel, server
self.server.set_channel(channel)
threading.Thread.__init__(self)
self.name = "SlaveThread (%s:%s)" % (self.server.address.host, self.server.address.port)
self.name = "SlaveThread (%s:%s)" % (
self.server.address.host, self.server.address.port)
def run(self):
self.server.serve_forever()
@ -81,6 +87,7 @@ class Master(object):
"""
Masters get and respond to messages from slaves.
"""
def __init__(self, server):
"""
server may be None if no server is needed.

25
libmproxy/dump.py
View File

@ -53,7 +53,7 @@ class Options(object):
def str_response(resp):
r = "%s %s"%(resp.code, resp.msg)
r = "%s %s" % (resp.code, resp.msg)
if resp.is_replay:
r = "[replay] " + r
return r
@ -64,7 +64,7 @@ def str_request(f, showhost):
c = f.client_conn.address.host
else:
c = "[replay]"
r = "%s %s %s"%(c, f.request.method, f.request.pretty_url(showhost))
r = "%s %s %s" % (c, f.request.method, f.request.pretty_url(showhost))
if f.request.stickycookie:
r = "[stickycookie] " + r
return r
@ -102,7 +102,7 @@ class DumpMaster(flow.FlowMaster):
try:
f = file(path, options.outfile[1])
self.start_stream(f, self.filt)
except IOError, v:
except IOError as v:
raise DumpError(v.strerror)
if options.replacements:
@ -140,7 +140,7 @@ class DumpMaster(flow.FlowMaster):
if options.rfile:
try:
self.load_flows_file(options.rfile)
except flow.FlowReadError, v:
except flow.FlowReadError as v:
self.add_event("Flow file corrupted.", "error")
raise DumpError(v)
@ -181,12 +181,18 @@ class DumpMaster(flow.FlowMaster):
if not utils.isBin(content):
try:
jsn = json.loads(content)
print(self.indent(4, json.dumps(jsn, indent=2)), file=self.outfile)
print(
self.indent(
4,
json.dumps(
jsn,
indent=2)),
file=self.outfile)
except ValueError:
print(self.indent(4, content), file=self.outfile)
else:
d = netlib.utils.hexdump(content)
d = "\n".join("%s\t%s %s"%i for i in d)
d = "\n".join("%s\t%s %s" % i for i in d)
print(self.indent(4, d), file=self.outfile)
if self.o.flow_detail >= 2:
print("", file=self.outfile)
@ -208,7 +214,12 @@ class DumpMaster(flow.FlowMaster):
sz = "(content missing)"
else:
sz = netlib.utils.pretty_size(len(f.response.content))
print(" << %s %s" % (str_response(f.response), sz), file=self.outfile)
print(
" << %s %s" %
(str_response(
f.response),
sz),
file=self.outfile)
self._print_message(f.response)
if f.error:

10
libmproxy/encoding.py
View File

@ -3,12 +3,14 @@
"""
from __future__ import absolute_import
import cStringIO
import gzip, zlib
import gzip
import zlib
__ALL__ = ["ENCODINGS"]
ENCODINGS = set(["identity", "gzip", "deflate"])
def decode(e, content):
encoding_map = {
"identity": identity,
@ -19,6 +21,7 @@ def decode(e, content):
return None
return encoding_map[e](content)
def encode(e, content):
encoding_map = {
"identity": identity,
@ -29,6 +32,7 @@ def encode(e, content):
return None
return encoding_map[e](content)
def identity(content):
"""
Returns content unchanged. Identity is the default value of
@ -36,6 +40,7 @@ def identity(content):
"""
return content
def decode_gzip(content):
gfile = gzip.GzipFile(fileobj=cStringIO.StringIO(content))
try:
@ -43,6 +48,7 @@ def decode_gzip(content):
except (IOError, EOFError):
return None
def encode_gzip(content):
s = cStringIO.StringIO()
gf = gzip.GzipFile(fileobj=s, mode='wb')
@ -50,6 +56,7 @@ def encode_gzip(content):
gf.close()
return s.getvalue()
def decode_deflate(content):
"""
Returns decompressed data for DEFLATE. Some servers may respond with
@ -67,6 +74,7 @@ def decode_deflate(content):
except zlib.error:
return None
def encode_deflate(content):
"""
Returns compressed content, always including zlib header and checksum.

72
libmproxy/filt.py
View File

@ -32,16 +32,17 @@
rex Equivalent to ~u rex
"""
from __future__ import absolute_import
import re, sys
import re
import sys
from .contrib import pyparsing as pp
from .protocol.http import decoded
class _Token:
def dump(self, indent=0, fp=sys.stdout):
print >> fp, "\t"*indent, self.__class__.__name__,
print >> fp, "\t" * indent, self.__class__.__name__,
if hasattr(self, "expr"):
print >> fp, "(%s)"%self.expr,
print >> fp, "(%s)" % self.expr,
print >> fp
@ -54,6 +55,7 @@ class _Action(_Token):
class FErr(_Action):
code = "e"
help = "Match error"
def __call__(self, f):
return True if f.error else False
@ -61,6 +63,7 @@ class FErr(_Action):
class FReq(_Action):
code = "q"
help = "Match request with no response"
def __call__(self, f):
if not f.response:
return True
@ -69,6 +72,7 @@ class FReq(_Action):
class FResp(_Action):
code = "s"
help = "Match response"
def __call__(self, f):
return True if f.response else False
@ -79,7 +83,7 @@ class _Rex(_Action):
try:
self.re = re.compile(self.expr)
except:
raise ValueError, "Cannot compile expression."
raise ValueError("Cannot compile expression.")
def _check_content_type(expr, o):
@ -100,6 +104,7 @@ class FAsset(_Action):
"image/.*",
"application/x-shockwave-flash"
]
def __call__(self, f):
if f.response:
for i in self.ASSET_TYPES:
@ -111,6 +116,7 @@ class FAsset(_Action):
class FContentType(_Rex):
code = "t"
help = "Content-type header"
def __call__(self, f):
if _check_content_type(self.expr, f.request):
return True
@ -122,6 +128,7 @@ class FContentType(_Rex):
class FRequestContentType(_Rex):
code = "tq"
help = "Request Content-Type header"
def __call__(self, f):
return _check_content_type(self.expr, f.request)
@ -129,6 +136,7 @@ class FRequestContentType(_Rex):
class FResponseContentType(_Rex):
code = "ts"
help = "Response Content-Type header"
def __call__(self, f):
if f.response:
return _check_content_type(self.expr, f.response)
@ -138,6 +146,7 @@ class FResponseContentType(_Rex):
class FHead(_Rex):
code = "h"
help = "Header"
def __call__(self, f):
if f.request.headers.match_re(self.expr):
return True
@ -149,6 +158,7 @@ class FHead(_Rex):
class FHeadRequest(_Rex):
code = "hq"
help = "Request header"
def __call__(self, f):
if f.request.headers.match_re(self.expr):
return True
@ -157,6 +167,7 @@ class FHeadRequest(_Rex):
class FHeadResponse(_Rex):
code = "hs"
help = "Response header"
def __call__(self, f):
if f.response and f.response.headers.match_re(self.expr):
return True
@ -165,6 +176,7 @@ class FHeadResponse(_Rex):
class FBod(_Rex):
code = "b"
help = "Body"
def __call__(self, f):
if f.request and f.request.content:
with decoded(f.request):
@ -180,6 +192,7 @@ class FBod(_Rex):
class FBodRequest(_Rex):
code = "bq"
help = "Request body"
def __call__(self, f):
if f.request and f.request.content:
with decoded(f.request):
@ -190,6 +203,7 @@ class FBodRequest(_Rex):
class FBodResponse(_Rex):
code = "bs"
help = "Response body"
def __call__(self, f):
if f.response and f.response.content:
with decoded(f.response):
@ -200,6 +214,7 @@ class FBodResponse(_Rex):
class FMethod(_Rex):
code = "m"
help = "Method"
def __call__(self, f):
return bool(re.search(self.expr, f.request.method, re.IGNORECASE))
@ -207,6 +222,7 @@ class FMethod(_Rex):
class FDomain(_Rex):
code = "d"
help = "Domain"
def __call__(self, f):
return bool(re.search(self.expr, f.request.host, re.IGNORECASE))
@ -215,6 +231,7 @@ class FUrl(_Rex):
code = "u"
help = "URL"
# FUrl is special, because it can be "naked".
@classmethod
def make(klass, s, loc, toks):
if len(toks) > 1:
@ -233,6 +250,7 @@ class _Int(_Action):
class FCode(_Int):
code = "c"
help = "HTTP response code"
def __call__(self, f):
if f.response and f.response.code == self.num:
return True
@ -243,9 +261,9 @@ class FAnd(_Token):
self.lst = lst
def dump(self, indent=0, fp=sys.stdout):
print >> fp, "\t"*indent, self.__class__.__name__
print >> fp, "\t" * indent, self.__class__.__name__
for i in self.lst:
i.dump(indent+1, fp)
i.dump(indent + 1, fp)
def __call__(self, f):
return all(i(f) for i in self.lst)
@ -256,9 +274,9 @@ class FOr(_Token):
self.lst = lst
def dump(self, indent=0, fp=sys.stdout):
print >> fp, "\t"*indent, self.__class__.__name__
print >> fp, "\t" * indent, self.__class__.__name__
for i in self.lst:
i.dump(indent+1, fp)
i.dump(indent + 1, fp)
def __call__(self, f):
return any(i(f) for i in self.lst)
@ -269,7 +287,7 @@ class FNot(_Token):
self.itm = itm[0]
def dump(self, indent=0, fp=sys.stdout):
print >> fp, "\t"*indent, self.__class__.__name__
print >> fp, "\t" * indent, self.__class__.__name__
self.itm.dump(indent + 1, fp)
def __call__(self, f):
@ -299,12 +317,14 @@ filt_rex = [
filt_int = [
FCode
]
def _make():
# Order is important - multi-char expressions need to come before narrow
# ones.
parts = []
for klass in filt_unary:
f = pp.Literal("~%s"%klass.code)
f = pp.Literal("~%s" % klass.code)
f.setParseAction(klass.make)
parts.append(f)
@ -313,12 +333,12 @@ def _make():
pp.QuotedString("\"", escChar='\\') |\
pp.QuotedString("'", escChar='\\')
for klass in filt_rex:
f = pp.Literal("~%s"%klass.code) + rex.copy()
f = pp.Literal("~%s" % klass.code) + rex.copy()
f.setParseAction(klass.make)
parts.append(f)
for klass in filt_int:
f = pp.Literal("~%s"%klass.code) + pp.Word(pp.nums)
f = pp.Literal("~%s" % klass.code) + pp.Word(pp.nums)
f.setParseAction(klass.make)
parts.append(f)
@ -328,14 +348,20 @@ def _make():
parts.append(f)
atom = pp.MatchFirst(parts)
expr = pp.operatorPrecedence(
atom,
[
(pp.Literal("!").suppress(), 1, pp.opAssoc.RIGHT, lambda x: FNot(*x)),
(pp.Literal("&").suppress(), 2, pp.opAssoc.LEFT, lambda x: FAnd(*x)),
(pp.Literal("|").suppress(), 2, pp.opAssoc.LEFT, lambda x: FOr(*x)),
]
)
expr = pp.operatorPrecedence(atom,
[(pp.Literal("!").suppress(),
1,
pp.opAssoc.RIGHT,
lambda x: FNot(*x)),
(pp.Literal("&").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FAnd(*x)),
(pp.Literal("|").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FOr(*x)),
])
expr = pp.OneOrMore(expr)
return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
bnf = _make()
@ -355,15 +381,15 @@ def parse(s):
help = []
for i in filt_unary:
help.append(
("~%s"%i.code, i.help)
("~%s" % i.code, i.help)
)
for i in filt_rex:
help.append(
("~%s regex"%i.code, i.help)
("~%s regex" % i.code, i.help)
)
for i in filt_int:
help.append(
("~%s int"%i.code, i.help)
("~%s int" % i.code, i.help)
)
help.sort()
help.extend(

79
libmproxy/flow.py
View File

@ -162,7 +162,8 @@ class StreamLargeBodies(object):
r.headers, is_request, flow.request.method, code
)
if not (0 <= expected_size <= self.max_size):
r.stream = r.stream or True # r.stream may already be a callable, which we want to preserve.
# r.stream may already be a callable, which we want to preserve.
r.stream = r.stream or True
class ClientPlaybackState:
@ -200,8 +201,16 @@ class ClientPlaybackState:
class ServerPlaybackState:
def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content,
ignore_payload_params, ignore_host):
def __init__(
self,
headers,
flows,
exit,
nopop,
ignore_params,
ignore_content,
ignore_payload_params,
ignore_host):
"""
headers: Case-insensitive list of request headers that should be
included in request-response matching.
@ -459,6 +468,7 @@ class FlowStore(FlowList):
Notifies the state that a flow has been updated.
The flow must be present in the state.
"""
if f in self:
for view in self.views:
view._update(f)
@ -531,7 +541,8 @@ class State(object):
def flow_count(self):
return len(self.flows)
# TODO: All functions regarding flows that don't cause side-effects should be moved into FlowStore.
# TODO: All functions regarding flows that don't cause side-effects should
# be moved into FlowStore.
def index(self, f):
return self.flows.index(f)
@ -659,7 +670,7 @@ class FlowMaster(controller.Master):
"""
try:
s = script.Script(command, self)
except script.ScriptError, v:
except script.ScriptError as v:
return v.args[0]
self.scripts.append(s)
@ -723,8 +734,17 @@ class FlowMaster(controller.Master):
def stop_client_playback(self):
self.client_playback = None
def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params,
ignore_content, ignore_payload_params, ignore_host):
def start_server_playback(
self,
flows,
kill,
headers,
exit,
nopop,
ignore_params,
ignore_content,
ignore_payload_params,
ignore_host):
"""
flows: List of flows.
kill: Boolean, should we kill requests not part of the replay?
@ -733,9 +753,15 @@ class FlowMaster(controller.Master):
ignore_payload_params: list of content params to ignore in server replay
ignore_host: true if request host should be ignored in server replay
"""
self.server_playback = ServerPlaybackState(headers, flows, exit, nopop,
ignore_params, ignore_content,
ignore_payload_params, ignore_host)
self.server_playback = ServerPlaybackState(
headers,
flows,
exit,
nopop,
ignore_params,
ignore_content,
ignore_payload_params,
ignore_host)
self.kill_nonreplay = kill
def stop_server_playback(self):
@ -792,16 +818,29 @@ class FlowMaster(controller.Master):
s = ServerConnection.from_state(dict(
address=dict(address=(host, port), use_ipv6=False),
state=[],
source_address=None, #source_address=dict(address=(host, port), use_ipv6=False),
source_address=None,
# source_address=dict(address=(host, port), use_ipv6=False),
cert=None,
sni=host,
ssl_established=True
))
f = http.HTTPFlow(c,s);
f = http.HTTPFlow(c, s)
headers = ODictCaseless()
req = http.HTTPRequest("absolute", method, scheme, host, port, path, (1, 1), headers, None,
None, None, None)
req = http.HTTPRequest(
"absolute",
method,
scheme,
host,
port,
path,
(1,
1),
headers,
None,
None,
None,
None)
f.request = req
return self.load_flow(f)
@ -812,7 +851,8 @@ class FlowMaster(controller.Master):
if self.server and self.server.config.mode == "reverse":
f.request.host, f.request.port = self.server.config.mode.dst[2:]
f.request.scheme = "https" if self.server.config.mode.dst[1] else "http"
f.request.scheme = "https" if self.server.config.mode.dst[
1] else "http"
f.reply = controller.DummyReply()
if f.request:
@ -839,7 +879,7 @@ class FlowMaster(controller.Master):
try:
f = file(path, "rb")
freader = FlowReader(f)
except IOError, v:
except IOError as v:
raise FlowReadError(v.strerror)
return self.load_flows(freader)
@ -880,7 +920,8 @@ class FlowMaster(controller.Master):
f.backup()
f.request.is_replay = True
if f.request.content:
f.request.headers["Content-Length"] = [str(len(f.request.content))]
f.request.headers[
"Content-Length"] = [str(len(f.request.content))]
f.response = None
f.error = None
self.process_new_request(f)
@ -1031,7 +1072,7 @@ class FlowReader:
"""
off = 0
try:
while 1:
while True:
data = tnetstring.load(self.fo)
if tuple(data["version"][:2]) != version.IVERSION[:2]:
v = ".".join(str(i) for i in data["version"])
@ -1040,7 +1081,7 @@ class FlowReader:
)
off = self.fo.tell()
yield handle.protocols[data["type"]]["flow"].from_state(data)
except ValueError, v:
except ValueError as v:
# Error is due to EOF
if self.fo.tell() == off and self.fo.read() == '':
return

2
libmproxy/main.py
View File

@ -70,7 +70,7 @@ def get_server(dummy_server, options):
else:
try:
return ProxyServer(options)
except ProxyServerError, v:
except ProxyServerError as v:
print(str(v), file=sys.stderr)
sys.exit(1)

13
libmproxy/onboarding/app.py
View File

@ -45,7 +45,10 @@ class PEM(tornado.web.RequestHandler):
def get(self):
p = os.path.join(self.request.master.server.config.cadir, self.filename)
self.set_header("Content-Type", "application/x-x509-ca-cert")
self.set_header("Content-Disposition", "inline; filename={}".format(self.filename))
self.set_header(
"Content-Disposition",
"inline; filename={}".format(
self.filename))
with open(p, "rb") as f:
self.write(f.read())
@ -59,7 +62,10 @@ class P12(tornado.web.RequestHandler):
def get(self):
p = os.path.join(self.request.master.server.config.cadir, self.filename)
self.set_header("Content-Type", "application/x-pkcs12")
self.set_header("Content-Disposition", "inline; filename={}".format(self.filename))
self.set_header(
"Content-Disposition",
"inline; filename={}".format(
self.filename))
with open(p, "rb") as f:
self.write(f.read())
@ -78,7 +84,6 @@ application = tornado.web.Application(
}
),
],
#debug=True
# debug=True
)
mapp = Adapter(application)

3
libmproxy/platform/linux.py
View File

@ -1,4 +1,5 @@
import socket, struct
import socket
import struct
# Python socket module does not have this constant
SO_ORIGINAL_DST = 80

3
libmproxy/platform/osx.py
View File

@ -21,6 +21,7 @@ class Resolver(object):
peer = csock.getpeername()
stxt = subprocess.check_output(self.STATECMD, stderr=subprocess.STDOUT)
if "sudo: a password is required" in stxt:
raise RuntimeError("Insufficient privileges to access pfctl. "
raise RuntimeError(
"Insufficient privileges to access pfctl. "
"See http://mitmproxy.org/doc/transparent/osx.html for details.")
return pf.lookup(peer[0], peer[1], stxt)

64
libmproxy/platform/windows.py
View File

@ -197,9 +197,12 @@ class TransparentProxy(object):
self.driver = WinDivert()
self.driver.register()
self.request_filter = custom_filter or " or ".join(("tcp.DstPort == %d" % p) for p in redirect_ports)
self.request_filter = custom_filter or " or ".join(
("tcp.DstPort == %d" %
p) for p in redirect_ports)
self.request_forward_handle = None
self.request_forward_thread = threading.Thread(target=self.request_forward)
self.request_forward_thread = threading.Thread(
target=self.request_forward)
self.request_forward_thread.daemon = True
self.addr_pid_map = dict()
@ -235,17 +238,25 @@ class TransparentProxy(object):
# Block all ICMP requests (which are sent on Windows by default).
# In layman's terms: If we don't do this, our proxy machine tells the client that it can directly connect to the
# real gateway if they are on the same network.
self.icmp_handle = self.driver.open_handle(filter="icmp", layer=Layer.NETWORK, flags=Flag.DROP)
self.icmp_handle = self.driver.open_handle(
filter="icmp",
layer=Layer.NETWORK,
flags=Flag.DROP)
self.response_handle = self.driver.open_handle(filter=self.response_filter, layer=Layer.NETWORK)
self.response_handle = self.driver.open_handle(
filter=self.response_filter,
layer=Layer.NETWORK)
self.response_thread.start()
if self.mode == "forward" or self.mode == "both":
self.request_forward_handle = self.driver.open_handle(filter=self.request_filter,
self.request_forward_handle = self.driver.open_handle(
filter=self.request_filter,
layer=Layer.NETWORK_FORWARD)
self.request_forward_thread.start()
if self.mode == "local" or self.mode == "both":
self.request_local_handle = self.driver.open_handle(filter=self.request_filter, layer=Layer.NETWORK)
self.request_local_handle = self.driver.open_handle(
filter=self.request_filter,
layer=Layer.NETWORK)
self.request_local_thread.start()
def shutdown(self):
@ -266,14 +277,17 @@ class TransparentProxy(object):
try:
raw_packet, metadata = handle.recv()
return self.driver.parse_packet(raw_packet), metadata
except WindowsError, e:
except WindowsError as e:
if e.winerror == 995:
return None, None
else:
raise
def fetch_pids(self):
ret = windll.iphlpapi.GetTcpTable2(byref(self.tcptable2), byref(self.tcptable2_size), 0)
ret = windll.iphlpapi.GetTcpTable2(
byref(
self.tcptable2), byref(
self.tcptable2_size), 0)
if ret == ERROR_INSUFFICIENT_BUFFER:
self.tcptable2 = MIB_TCPTABLE2(self.tcptable2_size.value)
self.fetch_pids()
@ -299,7 +313,8 @@ class TransparentProxy(object):
self.fetch_pids()
# If this fails, we most likely have a connection from an external client to
# a local server on 80/443. In this, case we always want to proxy the request.
# a local server on 80/443. In this, case we always want to proxy
# the request.
pid = self.addr_pid_map.get(client, None)
if pid not in self.trusted_pids:
@ -325,7 +340,8 @@ class TransparentProxy(object):
server = (packet.dst_addr, packet.dst_port)
if client in self.client_server_map:
del self.client_server_map[client] # Force re-add to mark as "newest" entry in the dict.
# Force re-add to mark as "newest" entry in the dict.
del self.client_server_map[client]
while len(self.client_server_map) > self.connection_cache_size:
self.client_server_map.popitem(False)
@ -335,7 +351,8 @@ class TransparentProxy(object):
metadata.direction = Direction.INBOUND
packet = self.driver.update_packet_checksums(packet)
# Use any handle thats on the NETWORK layer - request_local may be unavailable.
# Use any handle thats on the NETWORK layer - request_local may be
# unavailable.
self.response_handle.send((packet.raw, metadata))
def response(self):
@ -361,14 +378,31 @@ class TransparentProxy(object):
if __name__ == "__main__":
parser = configargparse.ArgumentParser(description="Windows Transparent Proxy")
parser.add_argument('--mode', choices=['forward', 'local', 'both'], default="both",
parser = configargparse.ArgumentParser(
description="Windows Transparent Proxy")
parser.add_argument(
'--mode',
choices=[
'forward',
'local',
'both'],
default="both",
help='redirection operation mode: "forward" to only redirect forwarded packets, '
'"local" to only redirect packets originating from the local machine')
group = parser.add_mutually_exclusive_group()
group.add_argument("--redirect-ports", nargs="+", type=int, default=[80, 443], metavar="80",
group.add_argument(
"--redirect-ports",
nargs="+",
type=int,
default=[
80,
443],
metavar="80",
help="ports that should be forwarded to the proxy")
group.add_argument("--custom-filter", default=None, metavar="WINDIVERT_FILTER",
group.add_argument(
"--custom-filter",
default=None,
metavar="WINDIVERT_FILTER",
help="Custom WinDivert interception rule.")
parser.add_argument("--proxy-addr", default=False,
help="Proxy Server Address")

5
libmproxy/protocol/handle.py
View File

@ -6,6 +6,7 @@ protocols = {
'tcp': dict(handler=tcp.TCPHandler)
}
def protocol_handler(protocol):
"""
@type protocol: str
@ -14,4 +15,6 @@ def protocol_handler(protocol):
if protocol in protocols:
return protocols[protocol]["handler"]
raise NotImplementedError("Unknown Protocol: %s" % protocol) # pragma: nocover
raise NotImplementedError(
"Unknown Protocol: %s" %
protocol) # pragma: nocover

93
libmproxy/protocol/http.py
View File

@ -305,7 +305,18 @@ class HTTPRequest(HTTPMessage):
@classmethod
def from_state(cls, state):
f = cls(None, None, None, None, None, None, None, None, None, None, None)
f = cls(
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None)
f.load_state(state)
return f
@ -315,7 +326,12 @@ class HTTPRequest(HTTPMessage):
)
@classmethod
def from_stream(cls, rfile, include_body=True, body_size_limit=None, wfile=None):
def from_stream(
cls,
rfile,
include_body=True,
body_size_limit=None,
wfile=None):
"""
Parse an HTTP request from a file stream
@ -403,7 +419,8 @@ class HTTPRequest(HTTPMessage):
self.host,
self.port)]
# If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header.
# If content is defined (i.e. not None or CONTENT_MISSING), we always
# add a content-length header.
if self.content or self.content == "":
headers["Content-Length"] = [str(len(self.content))]
@ -460,9 +477,9 @@ class HTTPRequest(HTTPMessage):
decode appropriately.
"""
if self.headers["accept-encoding"]:
self.headers["accept-encoding"] = [', '.join(
e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0]
)]
self.headers["accept-encoding"] = [
', '.join(
e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0])]
def update_host_header(self):
"""
@ -489,13 +506,22 @@ class HTTPRequest(HTTPMessage):
Returns an empty ODict if there is no data or the content-type
indicates non-form data.
"""
if self.content and self.headers.in_any("content-type", HDR_FORM_URLENCODED, True):
if self.content and self.headers.in_any(
"content-type",
HDR_FORM_URLENCODED,
True):
return odict.ODict(utils.urldecode(self.content))
return odict.ODict([])
def get_form_multipart(self):
if self.content and self.headers.in_any("content-type", HDR_FORM_MULTIPART, True):
return odict.ODict(utils.multipartdecode(self.headers, self.content))
if self.content and self.headers.in_any(
"content-type",
HDR_FORM_MULTIPART,
True):
return odict.ODict(
utils.multipartdecode(
self.headers,
self.content))
return odict.ODict([])
def set_form_urlencoded(self, odict):
@ -664,7 +690,14 @@ class HTTPResponse(HTTPMessage):
timestamp_end: Timestamp indicating when request transmission ended
"""
def __init__(self, httpversion, code, msg, headers, content, timestamp_start=None,
def __init__(
self,
httpversion,
code,
msg,
headers,
content,
timestamp_start=None,
timestamp_end=None):
assert isinstance(headers, odict.ODictCaseless) or headers is None
HTTPMessage.__init__(
@ -710,7 +743,12 @@ class HTTPResponse(HTTPMessage):
)
@classmethod
def from_stream(cls, rfile, request_method, include_body=True, body_size_limit=None):
def from_stream(
cls,
rfile,
request_method,
include_body=True,
body_size_limit=None):
"""
Parse an HTTP response from a file stream
"""
@ -760,7 +798,8 @@ class HTTPResponse(HTTPMessage):
if not preserve_transfer_encoding:
del headers['Transfer-Encoding']
# If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header.
# If content is defined (i.e. not None or CONTENT_MISSING), we always
# add a content-length header.
if self.content or self.content == "":
headers["Content-Length"] = [str(len(self.content))]
@ -1008,7 +1047,7 @@ class HTTPHandler(ProtocolHandler):
include_body=False
)
break
except (tcp.NetLibError, http.HttpErrorConnClosed), v:
except (tcp.NetLibError, http.HttpErrorConnClosed) as v:
self.c.log(
"error in server communication: %s" % repr(v),
level="debug"
@ -1079,7 +1118,8 @@ class HTTPHandler(ProtocolHandler):
if request_reply is None or request_reply == KILL:
raise KillSignal()
self.process_server_address(flow) # The inline script may have changed request.host
# The inline script may have changed request.host
self.process_server_address(flow)
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
@ -1090,7 +1130,9 @@ class HTTPHandler(ProtocolHandler):
# we can safely set it as the final attribute value here.
flow.server_conn = self.c.server_conn
self.c.log("response", "debug", [flow.response._assemble_first_line()])
self.c.log(
"response", "debug", [
flow.response._assemble_first_line()])
response_reply = self.c.channel.ask("response", flow)
if response_reply is None or response_reply == KILL:
raise KillSignal()
@ -1117,7 +1159,8 @@ class HTTPHandler(ProtocolHandler):
}
)
)
if not self.process_connect_request((flow.request.host, flow.request.port)):
if not self.process_connect_request(
(flow.request.host, flow.request.port)):
return False
# If the user has changed the target server on this connection,
@ -1130,7 +1173,7 @@ class HTTPHandler(ProtocolHandler):
http.HttpError,
proxy.ProxyError,
tcp.NetLibError,
), e:
) as e:
self.handle_error(e, flow)
except KillSignal:
self.c.log("Connection killed", "info")
@ -1226,7 +1269,8 @@ class HTTPHandler(ProtocolHandler):
# Determine .scheme, .host and .port attributes
# For absolute-form requests, they are directly given in the request.
# For authority-form requests, we only need to determine the request scheme.
# For relative-form requests, we need to determine host and port as well.
# For relative-form requests, we need to determine host and port as
# well.
if not request.scheme:
request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http"
if not request.host:
@ -1253,7 +1297,7 @@ class HTTPHandler(ProtocolHandler):
flow.server_conn = self.c.server_conn
self.c.establish_server_connection()
self.c.client_conn.send(
('HTTP/%s.%s 200 ' % (request.httpversion[0],request.httpversion[1])) +
('HTTP/%s.%s 200 ' % (request.httpversion[0], request.httpversion[1])) +
'Connection established\r\n' +
'Content-Length: 0\r\n' +
('Proxy-agent: %s\r\n' % self.c.config.server_version) +
@ -1372,9 +1416,14 @@ class HTTPHandler(ProtocolHandler):
semantics. Returns True, if so.
"""
close_connection = (
http.connection_close(flow.request.httpversion, flow.request.headers) or
http.connection_close(flow.response.httpversion, flow.response.headers) or
http.expected_http_body_size(flow.response.headers, False, flow.request.method,
http.connection_close(
flow.request.httpversion,
flow.request.headers) or http.connection_close(
flow.response.httpversion,
flow.response.headers) or http.expected_http_body_size(
flow.response.headers,
False,
flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:

13
libmproxy/protocol/primitives.py
View File

@ -24,6 +24,7 @@ class Error(stateobject.StateObject):
msg: Message describing the error
timestamp: Seconds since the epoch
"""
def __init__(self, msg, timestamp=None):
"""
@type msg: str
@ -59,6 +60,7 @@ class Flow(stateobject.StateObject):
A Flow is a collection of objects representing a single transaction.
This class is usually subclassed for each protocol, e.g. HTTPFlow.
"""
def __init__(self, type, client_conn, server_conn, live=None):
self.type = type
self.id = str(uuid.uuid4())
@ -165,12 +167,12 @@ class Flow(stateobject.StateObject):
master.handle_accept_intercept(self)
class ProtocolHandler(object):
"""
A ProtocolHandler implements an application-layer protocol, e.g. HTTP.
See: libmproxy.protocol.http.HTTPHandler
"""
def __init__(self, c):
self.c = c
"""@type: libmproxy.proxy.server.ConnectionHandler"""
@ -209,13 +211,20 @@ class LiveConnection(object):
interface with a live connection, without exposing the internals
of the ConnectionHandler.
"""
def __init__(self, c):
self.c = c
"""@type: libmproxy.proxy.server.ConnectionHandler"""
self._backup_server_conn = None
"""@type: libmproxy.proxy.connection.ServerConnection"""
def change_server(self, address, ssl=None, sni=None, force=False, persistent_change=False):
def change_server(
self,
address,
ssl=None,
sni=None,
force=False,
persistent_change=False):
"""
Change the server connection to the specified address.
@returns:

3
libmproxy/protocol/tcp.py
View File

@ -79,7 +79,8 @@ class TCPHandler(ProtocolHandler):
),
"info"
)
# Do not use dst.connection.send here, which may raise OpenSSL-specific errors.
# Do not use dst.connection.send here, which may raise
# OpenSSL-specific errors.
dst.send(contents)
else:
# socket.socket.send supports raw bytearrays/memoryviews

62
libmproxy/proxy/config.py
View File

@ -81,16 +81,27 @@ class ProxyConfig:
self.check_tcp = HostMatcher(tcp_hosts)
self.authenticator = authenticator
self.cadir = os.path.expanduser(cadir)
self.certstore = certutils.CertStore.from_store(self.cadir, CONF_BASENAME)
self.certstore = certutils.CertStore.from_store(
self.cadir,
CONF_BASENAME)
for spec, cert in certs:
self.certstore.add_cert_file(spec, cert)
self.certforward = certforward
self.openssl_method_client, self.openssl_options_client = version_to_openssl(ssl_version_client)
self.openssl_method_server, self.openssl_options_server = version_to_openssl(ssl_version_server)
self.openssl_method_client, self.openssl_options_client = version_to_openssl(
ssl_version_client)
self.openssl_method_server, self.openssl_options_server = version_to_openssl(
ssl_version_server)
self.ssl_ports = ssl_ports
sslversion_choices = ("all", "secure", "SSLv2", "SSLv3", "TLSv1", "TLSv1_1", "TLSv1_2")
sslversion_choices = (
"all",
"secure",
"SSLv2",
"SSLv3",
"TLSv1",
"TLSv1_1",
"TLSv1_2")
def version_to_openssl(version):
@ -119,7 +130,8 @@ def process_proxy_options(parser, options):
if options.transparent_proxy:
c += 1
if not platform.resolver:
return parser.error("Transparent mode not supported on this platform.")
return parser.error(
"Transparent mode not supported on this platform.")
mode = "transparent"
if options.socks_proxy:
c += 1
@ -133,28 +145,33 @@ def process_proxy_options(parser, options):
mode = "upstream"
upstream_server = options.upstream_proxy
if c > 1:
return parser.error("Transparent, SOCKS5, reverse and upstream proxy mode "
return parser.error(
"Transparent, SOCKS5, reverse and upstream proxy mode "
"are mutually exclusive.")
if options.clientcerts:
options.clientcerts = os.path.expanduser(options.clientcerts)
if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts):
if not os.path.exists(
options.clientcerts) or not os.path.isdir(
options.clientcerts):
return parser.error(
"Client certificate directory does not exist or is not a directory: %s" % options.clientcerts
)
"Client certificate directory does not exist or is not a directory: %s" %
options.clientcerts)
if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd):
if options.auth_singleuser:
if len(options.auth_singleuser.split(':')) != 2:
return parser.error("Invalid single-user specification. Please use the format username:password")
return parser.error(
"Invalid single-user specification. Please use the format username:password")
username, password = options.auth_singleuser.split(':')
password_manager = http_auth.PassManSingleUser(username, password)
elif options.auth_nonanonymous:
password_manager = http_auth.PassManNonAnon()
elif options.auth_htpasswd:
try:
password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd)
except ValueError, v:
password_manager = http_auth.PassManHtpasswd(
options.auth_htpasswd)
except ValueError as v:
return parser.error(v.message)
authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")
else:
@ -203,15 +220,18 @@ def process_proxy_options(parser, options):
def ssl_option_group(parser):
group = parser.add_argument_group("SSL")
group.add_argument(
"--cert", dest='certs', default=[], type=str,
metavar="SPEC", action="append",
"--cert",
dest='certs',
default=[],
type=str,
metavar="SPEC",
action="append",
help='Add an SSL certificate. SPEC is of the form "[domain=]path". '
'The domain may include a wildcard, and is equal to "*" if not specified. '
'The file at path is a certificate in PEM format. If a private key is included in the PEM, '
'it is used, else the default key in the conf dir is used. '
'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. '
'Can be passed multiple times.'
)
'Can be passed multiple times.')
group.add_argument(
"--cert-forward", action="store_true",
dest="certforward", default=False,
@ -238,11 +258,15 @@ def ssl_option_group(parser):
help="Don't connect to upstream server to look up certificate details."
)
group.add_argument(
"--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS),
"--ssl-port",
action="append",
type=int,
dest="ssl_ports",
default=list(TRANSPARENT_SSL_PORTS),
metavar="PORT",
help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. "
"Defaults to %s." % str(TRANSPARENT_SSL_PORTS)
)
"Defaults to %s." %
str(TRANSPARENT_SSL_PORTS))
group.add_argument(
"--ssl-version-client", dest="ssl_version_client",
default="secure", action="store",

30
libmproxy/proxy/connection.py
View File

@ -7,7 +7,9 @@ from .. import stateobject, utils
class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
def __init__(self, client_connection, address, server):
if client_connection: # Eventually, this object is restored from state. We don't have a connection then.
# Eventually, this object is restored from state. We don't have a
# connection then.
if client_connection:
tcp.BaseHandler.__init__(self, client_connection, address, server)
else:
self.connection = None
@ -39,15 +41,18 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
def get_state(self, short=False):
d = super(ClientConnection, self).get_state(short)
d.update(
address={"address": self.address(), "use_ipv6": self.address.use_ipv6},
clientcert=self.cert.to_pem() if self.clientcert else None
)
address={
"address": self.address(),
"use_ipv6": self.address.use_ipv6},
clientcert=self.cert.to_pem() if self.clientcert else None)
return d
def load_state(self, state):
super(ClientConnection, self).load_state(state)
self.address = tcp.Address(**state["address"]) if state["address"] else None
self.clientcert = certutils.SSLCert.from_pem(state["clientcert"]) if state["clientcert"] else None
self.address = tcp.Address(
**state["address"]) if state["address"] else None
self.clientcert = certutils.SSLCert.from_pem(
state["clientcert"]) if state["clientcert"] else None
def copy(self):
return copy.copy(self)
@ -122,9 +127,12 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
def load_state(self, state):
super(ServerConnection, self).load_state(state)
self.address = tcp.Address(**state["address"]) if state["address"] else None
self.source_address = tcp.Address(**state["source_address"]) if state["source_address"] else None
self.cert = certutils.SSLCert.from_pem(state["cert"]) if state["cert"] else None
self.address = tcp.Address(
**state["address"]) if state["address"] else None
self.source_address = tcp.Address(
**state["source_address"]) if state["source_address"] else None
self.cert = certutils.SSLCert.from_pem(
state["cert"]) if state["cert"] else None
@classmethod
def from_state(cls, state):
@ -147,7 +155,9 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
def establish_ssl(self, clientcerts, sni, **kwargs):
clientcert = None
if clientcerts:
path = os.path.join(clientcerts, self.address.host.encode("idna")) + ".pem"
path = os.path.join(
clientcerts,
self.address.host.encode("idna")) + ".pem"
if os.path.exists(path):
clientcert = path
self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs)

13
libmproxy/proxy/primitives.py
View File

@ -1,6 +1,7 @@
from __future__ import absolute_import
from netlib import socks
class ProxyError(Exception):
def __init__(self, code, message, headers=None):
super(ProxyError, self).__init__(message)
@ -61,7 +62,7 @@ class TransparentProxyMode(ProxyMode):
def get_upstream_server(self, client_conn):
try:
dst = self.resolver.original_addr(client_conn.connection)
except Exception, e:
except Exception as e:
raise ProxyError(502, "Transparent mode failure: %s" % str(e))
if dst[1] in self.sslports:
@ -87,7 +88,9 @@ class Socks5ProxyMode(ProxyMode):
guess = ""
raise socks.SocksError(
socks.REP.GENERAL_SOCKS_SERVER_FAILURE,
guess + "Invalid SOCKS version. Expected 0x05, got 0x%x" % msg.ver)
guess +
"Invalid SOCKS version. Expected 0x05, got 0x%x" %
msg.ver)
def get_upstream_server(self, client_conn):
try:
@ -117,13 +120,15 @@ class Socks5ProxyMode(ProxyMode):
"mitmproxy only supports SOCKS5 CONNECT."
)
# We do not connect here yet, as the clientconnect event has not been handled yet.
# We do not connect here yet, as the clientconnect event has not
# been handled yet.
connect_reply = socks.Message(
socks.VERSION.SOCKS5,
socks.REP.SUCCEEDED,
socks.ATYP.DOMAINNAME,
client_conn.address # dummy value, we don't have an upstream connection yet.
# dummy value, we don't have an upstream connection yet.
client_conn.address
)
connect_reply.to_file(client_conn.wfile)
client_conn.wfile.flush()

95
libmproxy/proxy/server.py
View File

@ -34,7 +34,7 @@ class ProxyServer(tcp.TCPServer):
self.config = config
try:
tcp.TCPServer.__init__(self, (config.host, config.port))
except socket.error, v:
except socket.error as v:
raise ProxyServerError('Error starting proxy server: ' + repr(v))
self.channel = None
@ -46,16 +46,30 @@ class ProxyServer(tcp.TCPServer):
self.channel = channel
def handle_client_connection(self, conn, client_address):
h = ConnectionHandler(self.config, conn, client_address, self, self.channel)
h = ConnectionHandler(
self.config,
conn,
client_address,
self,
self.channel)
h.handle()
h.finish()
class ConnectionHandler:
def __init__(self, config, client_connection, client_address, server, channel):
def __init__(
self,
config,
client_connection,
client_address,
server,
channel):
self.config = config
"""@type: libmproxy.proxy.config.ProxyConfig"""
self.client_conn = ClientConnection(client_connection, client_address, server)
self.client_conn = ClientConnection(
client_connection,
client_address,
server)
"""@type: libmproxy.proxy.connection.ClientConnection"""
self.server_conn = None
"""@type: libmproxy.proxy.connection.ServerConnection"""
@ -70,17 +84,23 @@ class ConnectionHandler:
# Can we already identify the target server and connect to it?
client_ssl, server_ssl = False, False
conn_kwargs = dict()
upstream_info = self.config.mode.get_upstream_server(self.client_conn)
upstream_info = self.config.mode.get_upstream_server(
self.client_conn)
if upstream_info:
self.set_server_address(upstream_info[2:])
client_ssl, server_ssl = upstream_info[:2]
if self.config.check_ignore(self.server_conn.address):
self.log("Ignore host: %s:%s" % self.server_conn.address(), "info")
self.log(
"Ignore host: %s:%s" %
self.server_conn.address(),
"info")
self.conntype = "tcp"
conn_kwargs["log"] = False
client_ssl, server_ssl = False, False
else:
pass # No upstream info from the metadata: upstream info in the protocol (e.g. HTTP absolute-form)
# No upstream info from the metadata: upstream info in the
# protocol (e.g. HTTP absolute-form)
pass
self.channel.ask("clientconnect", self)
@ -92,11 +112,17 @@ class ConnectionHandler:
self.establish_ssl(client=client_ssl, server=server_ssl)
if self.config.check_tcp(self.server_conn.address):
self.log("Generic TCP mode for host: %s:%s" % self.server_conn.address(), "info")
self.log(
"Generic TCP mode for host: %s:%s" %
self.server_conn.address(),
"info")
self.conntype = "tcp"
# Delegate handling to the protocol handler
protocol_handler(self.conntype)(self, **conn_kwargs).handle_messages()
protocol_handler(
self.conntype)(
self,
**conn_kwargs).handle_messages()
self.log("clientdisconnect", "info")
self.channel.tell("clientdisconnect", self)
@ -104,7 +130,8 @@ class ConnectionHandler:
except ProxyError as e:
protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e)
except Exception:
import traceback, sys
import traceback
import sys
self.log(traceback.format_exc(), "error")
print >> sys.stderr, traceback.format_exc()
@ -112,7 +139,8 @@ class ConnectionHandler:
print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy"
finally:
# Make sure that we close the server connection in any case.
# The client connection is closed by the ProxyServer and does not have be handled here.
# The client connection is closed by the ProxyServer and does not
# have be handled here.
self.del_server_connection()
def del_server_connection(self):
@ -122,8 +150,10 @@ class ConnectionHandler:
if self.server_conn and self.server_conn.connection:
self.server_conn.finish()
self.server_conn.close()
self.log("serverdisconnect", "debug", ["%s:%s" % (self.server_conn.address.host,
self.server_conn.address.port)])
self.log(
"serverdisconnect", "debug", [
"%s:%s" %
(self.server_conn.address.host, self.server_conn.address.port)])
self.channel.tell("serverdisconnect", self)
self.server_conn = None
@ -141,7 +171,9 @@ class ConnectionHandler:
if self.server_conn:
self.del_server_connection()
self.log("Set new server address: %s:%s" % (address.host, address.port), "debug")
self.log(
"Set new server address: %s:%s" %
(address.host, address.port), "debug")
self.server_conn = ServerConnection(address)
def establish_server_connection(self, ask=True):
@ -155,12 +187,16 @@ class ConnectionHandler:
"""
if self.server_conn.connection:
return
self.log("serverconnect", "debug", ["%s:%s" % self.server_conn.address()[:2]])
self.log(
"serverconnect", "debug", [
"%s:%s" %
self.server_conn.address()[
:2]])
if ask:
self.channel.ask("serverconnect", self)
try:
self.server_conn.connect()
except tcp.NetLibError, v:
except tcp.NetLibError as v:
raise ProxyError(502, v)
def establish_ssl(self, client=False, server=False, sni=None):
@ -237,7 +273,8 @@ class ConnectionHandler:
self.server_conn.state = state
# Receiving new_sni where had_ssl is False is a weird case that happens when the workaround for
# https://github.com/mitmproxy/mitmproxy/issues/427 is active. In this case, we want to establish SSL as well.
# https://github.com/mitmproxy/mitmproxy/issues/427 is active. In this
# case, we want to establish SSL as well.
if had_ssl or new_sni:
self.establish_ssl(server=True, sni=sni)
@ -246,8 +283,10 @@ class ConnectionHandler:
def log(self, msg, level, subs=()):
msg = [
"%s:%s: %s" % (self.client_conn.address.host, self.client_conn.address.port, msg)
]
"%s:%s: %s" %
(self.client_conn.address.host,
self.client_conn.address.port,
msg)]
for i in subs:
msg.append(" -> " + i)
msg = "\n".join(msg)
@ -255,11 +294,13 @@ class ConnectionHandler:
def find_cert(self):
if self.config.certforward and self.server_conn.ssl_established:
return self.server_conn.cert, self.config.certstore.gen_pkey(self.server_conn.cert), None
return self.server_conn.cert, self.config.certstore.gen_pkey(
self.server_conn.cert), None
else:
host = self.server_conn.address.host
sans = []
if self.server_conn.ssl_established and (not self.config.no_upstream_cert):
if self.server_conn.ssl_established and (
not self.config.no_upstream_cert):
upstream_cert = self.server_conn.cert
sans.extend(upstream_cert.altnames)
if upstream_cert.cn:
@ -291,8 +332,11 @@ class ConnectionHandler:
# - We established SSL with the server previously
# - We initially wanted to establish SSL with the server,
# but the server refused to negotiate without SNI.
if self.server_conn.ssl_established or hasattr(self.server_conn, "may_require_sni"):
self.server_reconnect(sni) # reconnect to upstream server with SNI
if self.server_conn.ssl_established or hasattr(
self.server_conn,
"may_require_sni"):
# reconnect to upstream server with SNI
self.server_reconnect(sni)
# Now, change client context to reflect changed certificate:
cert, key, chain_file = self.find_cert()
new_context = self.client_conn.create_ssl_context(
@ -308,4 +352,7 @@ class ConnectionHandler:
# make dang sure it doesn't happen.
except: # pragma: no cover
import traceback
self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error")
self.log(
"Error in handle_sni:\r\n" +
traceback.format_exc(),
"error")

33
libmproxy/script.py
View File

@ -1,7 +1,11 @@
from __future__ import absolute_import
import os, traceback, threading, shlex
import os
import traceback
import threading
import shlex
from . import controller
class ScriptError(Exception):
pass
@ -56,6 +60,7 @@ class Script:
s = Script(argv, master)
s.load()
"""
def __init__(self, command, master):
self.command = command
self.argv = self.parse_command(command)
@ -73,9 +78,11 @@ class Script:
args = shlex.split(command)
args[0] = os.path.expanduser(args[0])
if not os.path.exists(args[0]):
raise ScriptError(("Script file not found: %s.\r\n"
raise ScriptError(
("Script file not found: %s.\r\n"
"If you script path contains spaces, "
"make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") % args[0])
"make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") %
args[0])
elif not os.path.isfile(args[0]):
raise ScriptError("Not a file: %s" % args[0])
return args
@ -90,7 +97,7 @@ class Script:
ns = {}
try:
execfile(self.argv[0], ns, ns)
except Exception, v:
except Exception as v:
raise ScriptError(traceback.format_exc(v))
self.ns = ns
r = self.run("start", self.argv)
@ -114,7 +121,7 @@ class Script:
if f:
try:
return (True, f(self.ctx, *args, **kwargs))
except Exception, v:
except Exception as v:
return (False, (v, traceback.format_exc(v)))
else:
return (False, None)
@ -133,7 +140,7 @@ class ReplyProxy(object):
return
self.original_reply(*args, **kwargs)
def __getattr__ (self, k):
def __getattr__(self, k):
return getattr(self.original_reply, k)
@ -145,7 +152,8 @@ def _handle_concurrent_reply(fn, o, *args, **kwargs):
def run():
fn(*args, **kwargs)
reply_proxy() # If the script did not call .reply(), we have to do it now.
# If the script did not call .reply(), we have to do it now.
reply_proxy()
ScriptThread(target=run).start()
@ -154,8 +162,15 @@ class ScriptThread(threading.Thread):
def concurrent(fn):
if fn.func_name in ("request", "response", "error", "clientconnect", "serverconnect", "clientdisconnect"):
if fn.func_name in (
"request",
"response",
"error",
"clientconnect",
"serverconnect",
"clientdisconnect"):
def _concurrent(ctx, obj):
_handle_concurrent_reply(fn, obj, ctx, obj)
return _concurrent
raise NotImplementedError("Concurrent decorator not supported for this method.")
raise NotImplementedError(
"Concurrent decorator not supported for this method.")

92
libmproxy/tnetstring.py
View File

@ -72,13 +72,14 @@ __ver_major__ = 0
__ver_minor__ = 2
__ver_patch__ = 0
__ver_sub__ = ""
__version__ = "%d.%d.%d%s" % (__ver_major__,__ver_minor__,__ver_patch__,__ver_sub__)
__version__ = "%d.%d.%d%s" % (
__ver_major__, __ver_minor__, __ver_patch__, __ver_sub__)
from collections import deque
def dumps(value,encoding=None):
def dumps(value, encoding=None):
"""dumps(object,encoding=None) -> string
This function dumps a python object as a tnetstring.
@ -90,21 +91,21 @@ def dumps(value,encoding=None):
# consider the _gdumps() function instead; it's a standard top-down
# generator that's simpler to understand but much less efficient.
q = deque()
_rdumpq(q,0,value,encoding)
_rdumpq(q, 0, value, encoding)
return "".join(q)
def dump(value,file,encoding=None):
def dump(value, file, encoding=None):
"""dump(object,file,encoding=None)
This function dumps a python object as a tnetstring and writes it to
the given file.
"""
file.write(dumps(value,encoding))
file.write(dumps(value, encoding))
file.flush()
def _rdumpq(q,size,value,encoding=None):
def _rdumpq(q, size, value, encoding=None):
"""Dump value as a tnetstring, to a deque instance, last chunks first.
This function generates the tnetstring representation of the given value,
@ -129,7 +130,7 @@ def _rdumpq(q,size,value,encoding=None):
if value is False:
write("5:false!")
return size + 8
if isinstance(value,(int,long)):
if isinstance(value, (int, long)):
data = str(value)
ldata = len(data)
span = str(ldata)
@ -138,7 +139,7 @@ def _rdumpq(q,size,value,encoding=None):
write(":")
write(span)
return size + 2 + len(span) + ldata
if isinstance(value,(float,)):
if isinstance(value, (float,)):
# Use repr() for float rather than str().
# It round-trips more accurately.
# Probably unnecessary in later python versions that
@ -151,7 +152,7 @@ def _rdumpq(q,size,value,encoding=None):
write(":")
write(span)
return size + 2 + len(span) + ldata
if isinstance(value,str):
if isinstance(value, str):
lvalue = len(value)
span = str(lvalue)
write(",")
@ -159,26 +160,26 @@ def _rdumpq(q,size,value,encoding=None):
write(":")
write(span)
return size + 2 + len(span) + lvalue
if isinstance(value,(list,tuple,)):
if isinstance(value, (list, tuple,)):
write("]")
init_size = size = size + 1
for item in reversed(value):
size = _rdumpq(q,size,item,encoding)
size = _rdumpq(q, size, item, encoding)
span = str(size - init_size)
write(":")
write(span)
return size + 1 + len(span)
if isinstance(value,dict):
if isinstance(value, dict):
write("}")
init_size = size = size + 1
for (k,v) in value.iteritems():
size = _rdumpq(q,size,v,encoding)
size = _rdumpq(q,size,k,encoding)
for (k, v) in value.iteritems():
size = _rdumpq(q, size, v, encoding)
size = _rdumpq(q, size, k, encoding)
span = str(size - init_size)
write(":")
write(span)
return size + 1 + len(span)
if isinstance(value,unicode):
if isinstance(value, unicode):
if encoding is None:
raise ValueError("must specify encoding to dump unicode strings")
value = value.encode(encoding)
@ -192,7 +193,7 @@ def _rdumpq(q,size,value,encoding=None):
raise ValueError("unserializable object")
def _gdumps(value,encoding):
def _gdumps(value, encoding):
"""Generate fragments of value dumped as a tnetstring.
This is the naive dumping algorithm, implemented as a generator so that
@ -207,24 +208,24 @@ def _gdumps(value,encoding):
yield "4:true!"
elif value is False:
yield "5:false!"
elif isinstance(value,(int,long)):
elif isinstance(value, (int, long)):
data = str(value)
yield str(len(data))
yield ":"
yield data
yield "#"
elif isinstance(value,(float,)):
elif isinstance(value, (float,)):
data = repr(value)
yield str(len(data))
yield ":"
yield data
yield "^"
elif isinstance(value,(str,)):
elif isinstance(value, (str,)):
yield str(len(value))
yield ":"
yield value
yield ","
elif isinstance(value,(list,tuple,)):
elif isinstance(value, (list, tuple,)):
sub = []
for item in value:
sub.extend(_gdumps(item))
@ -233,9 +234,9 @@ def _gdumps(value,encoding):
yield ":"
yield sub
yield "]"
elif isinstance(value,(dict,)):
elif isinstance(value, (dict,)):
sub = []
for (k,v) in value.iteritems():
for (k, v) in value.iteritems():
sub.extend(_gdumps(k))
sub.extend(_gdumps(v))
sub = "".join(sub)
@ -243,7 +244,7 @@ def _gdumps(value,encoding):
yield ":"
yield sub
yield "}"
elif isinstance(value,(unicode,)):
elif isinstance(value, (unicode,)):
if encoding is None:
raise ValueError("must specify encoding to dump unicode strings")
value = value.encode(encoding)
@ -255,7 +256,7 @@ def _gdumps(value,encoding):
raise ValueError("unserializable object")
def loads(string,encoding=None):
def loads(string, encoding=None):
"""loads(string,encoding=None) -> object
This function parses a tnetstring into a python object.
@ -263,10 +264,10 @@ def loads(string,encoding=None):
# No point duplicating effort here. In the C-extension version,
# loads() is measurably faster then pop() since it can avoid
# the overhead of building a second string.
return pop(string,encoding)[0]
return pop(string, encoding)[0]
def load(file,encoding=None):
def load(file, encoding=None):
"""load(file,encoding=None) -> object
This function reads a tnetstring from a file and parses it into a
@ -324,21 +325,20 @@ def load(file,encoding=None):
if type == "]":
l = []
while data:
(item,data) = pop(data,encoding)
(item, data) = pop(data, encoding)
l.append(item)
return l
if type == "}":
d = {}
while data:
(key,data) = pop(data,encoding)
(val,data) = pop(data,encoding)
(key, data) = pop(data, encoding)
(val, data) = pop(data, encoding)
d[key] = val
return d
raise ValueError("unknown type tag")
def pop(string,encoding=None):
def pop(string, encoding=None):
"""pop(string,encoding=None) -> (object, remain)
This function parses a tnetstring into a python object.
@ -347,12 +347,12 @@ def pop(string,encoding=None):
"""
# Parse out data length, type and remaining string.
try:
(dlen,rest) = string.split(":",1)
(dlen, rest) = string.split(":", 1)
dlen = int(dlen)
except ValueError:
raise ValueError("not a tnetstring: missing or invalid length prefix")
try:
(data,type,remain) = (rest[:dlen],rest[dlen],rest[dlen+1:])
(data, type, remain) = (rest[:dlen], rest[dlen], rest[dlen + 1:])
except IndexError:
# This fires if len(rest) < dlen, meaning we don't need
# to further validate that data is the right length.
@ -360,40 +360,40 @@ def pop(string,encoding=None):
# Parse the data based on the type tag.
if type == ",":
if encoding is not None:
return (data.decode(encoding),remain)
return (data,remain)
return (data.decode(encoding), remain)
return (data, remain)
if type == "#":
try:
return (int(data),remain)
return (int(data), remain)
except ValueError:
raise ValueError("not a tnetstring: invalid integer literal")
if type == "^":
try:
return (float(data),remain)
return (float(data), remain)
except ValueError:
raise ValueError("not a tnetstring: invalid float literal")
if type == "!":
if data == "true":
return (True,remain)
return (True, remain)
elif data == "false":
return (False,remain)
return (False, remain)
else:
raise ValueError("not a tnetstring: invalid boolean literal")
if type == "~":
if data:
raise ValueError("not a tnetstring: invalid null literal")
return (None,remain)
return (None, remain)
if type == "]":
l = []
while data:
(item,data) = pop(data,encoding)
(item, data) = pop(data, encoding)
l.append(item)
return (l,remain)
return (l, remain)
if type == "}":
d = {}
while data:
(key,data) = pop(data,encoding)
(val,data) = pop(data,encoding)
(key, data) = pop(data, encoding)
(val, data) = pop(data, encoding)
d[key] = val
return (d,remain)
return (d, remain)
raise ValueError("unknown type tag")

27
libmproxy/utils.py
View File

@ -1,8 +1,14 @@
from __future__ import absolute_import
import os, datetime, urllib, re
import time, functools, cgi
import os
import datetime
import urllib
import re
import time
import functools
import cgi
import json
def timestamp():
"""
Returns a serializable UTC timestamp.
@ -91,7 +97,7 @@ def multipartdecode(hdrs, content):
match = rx.search(parts[1])
if match:
key = match.group(1)
value = "".join(parts[3+parts[2:].index(""):])
value = "".join(parts[3 + parts[2:].index(""):])
r.append((key, value))
return r
return []
@ -107,8 +113,8 @@ def pretty_duration(secs):
for limit, formatter in formatters:
if secs >= limit:
return formatter.format(secs)
#less than 1 sec
return "{:.0f}ms".format(secs*1000)
# less than 1 sec
return "{:.0f}ms".format(secs * 1000)
class Data:
@ -126,7 +132,7 @@ class Data:
"""
fullpath = os.path.join(self.dirname, path)
if not os.path.exists(fullpath):
raise ValueError, "dataPath: %s does not exist."%fullpath
raise ValueError("dataPath: %s does not exist." % fullpath)
return fullpath
pkg_data = Data(__name__)
@ -135,6 +141,7 @@ class LRUCache:
"""
A simple LRU cache for generated values.
"""
def __init__(self, size=100):
self.size = size
self.cache = {}
@ -147,7 +154,7 @@ class LRUCache:
*args: A list of immutable arguments, used to establish identiy by
*the cache, and passed to gen to generate values.
"""
if self.cache.has_key(args):
if args in self.cache:
self.cacheList.remove(args)
self.cacheList.insert(0, args)
return self.cache[args]
@ -195,14 +202,14 @@ def hostport(scheme, host, port):
if (port, scheme) in [(80, "http"), (443, "https")]:
return host
else:
return "%s:%s"%(host, port)
return "%s:%s" % (host, port)
def unparse_url(scheme, host, port, path=""):
"""
Returns a URL string, constructed from the specified compnents.
"""
return "%s://%s%s"%(scheme, hostport(scheme, host, port), path)
return "%s://%s%s" % (scheme, hostport(scheme, host, port), path)
def clean_hanging_newline(t):
@ -243,7 +250,7 @@ def parse_size(s):
try:
return int(s) * mult
except ValueError:
raise ValueError("Invalid size specification: %s"%s)
raise ValueError("Invalid size specification: %s" % s)
def safe_subn(pattern, repl, target, *args, **kwargs):

2
libmproxy/version.py
View File

@ -1,4 +1,4 @@
IVERSION = (0, 12, 0)
IVERSION = (0, 12, 1)
VERSION = ".".join(str(i) for i in IVERSION)
MINORVERSION = ".".join(str(i) for i in IVERSION[:2])
NAME = "mitmproxy"

9
libmproxy/web/__init__.py
View File

@ -79,6 +79,7 @@ class WebState(flow.State):
data=[]
)
class Options(object):
attributes = [
"app",
@ -128,15 +129,14 @@ class WebMaster(flow.FlowMaster):
if options.rfile:
try:
self.load_flows_file(options.rfile)
except flow.FlowReadError, v:
except flow.FlowReadError as v:
self.add_event(
"Could not read flow file: %s"%v,
"Could not read flow file: %s" % v,
"error"
)
if self.options.app:
self.start_app(self.options.app_host, self.options.app_port)
def tick(self):
flow.FlowMaster.tick(self, self.masterq, timeout=0)
@ -157,7 +157,8 @@ class WebMaster(flow.FlowMaster):
self.shutdown()
def _process_flow(self, f):
if self.state.intercept and self.state.intercept(f) and not f.request.is_replay:
if self.state.intercept and self.state.intercept(
f) and not f.request.is_replay:
f.intercept(self)
else:
f.reply()

7
libmproxy/web/app.py
View File

@ -27,7 +27,8 @@ class RequestHandler(tornado.web.RequestHandler):
@property
def json(self):
if not self.request.headers.get("Content-Type").startswith("application/json"):
if not self.request.headers.get(
"Content-Type").startswith("application/json"):
return None
return json.loads(self.request.body)
@ -67,8 +68,10 @@ class FiltHelp(RequestHandler):
commands=filt.help
))
class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler):
connections = None # raise an error if inherited class doesn't specify its own instance.
# raise an error if inherited class doesn't specify its own instance.
connections = None
def open(self):
self.connections.add(self)

4
release/mitmdump.spec
View File

@ -16,6 +16,10 @@ a.datas += Tree(
"./libmproxy/onboarding/static",
prefix="libmproxy/onboarding/static"
)
a.datas += Tree(
"../venv.mitmproxy/lib/python2.7/site-packages/cryptography/hazmat/bindings/openssl/src",
prefix = "cryptography/hazmat/bindings/openssl/src"
)
pyz = PYZ(a.pure)
exe = EXE(pyz,
a.scripts,

4
release/mitmproxy.spec
View File

@ -16,6 +16,10 @@ a.datas += Tree(
"./libmproxy/onboarding/static",
prefix="libmproxy/onboarding/static"
)
a.datas += Tree(
"../venv.mitmproxy/lib/python2.7/site-packages/cryptography/hazmat/bindings/openssl/src",
prefix = "cryptography/hazmat/bindings/openssl/src"
)
pyz = PYZ(a.pure)
exe = EXE(pyz,
a.scripts,

4
release/mitmweb.spec
View File

@ -24,6 +24,10 @@ a.datas += Tree(
"./libmproxy/web/static",
prefix="libmproxy/web/static"
)
a.datas += Tree(
"../venv.mitmproxy/lib/python2.7/site-packages/cryptography/hazmat/bindings/openssl/src",
prefix = "cryptography/hazmat/bindings/openssl/src"
)
pyz = PYZ(a.pure)
exe = EXE(pyz,
a.scripts,

3
release/release-checklist
View File

@ -5,6 +5,9 @@
netlib/netlib/version.py
pathod/libpathod/version.py
- Ensure that the website style assets have been compiled for production, and
synced to the docs.
- Render the docs:
cshape doc-src doc

9
setup.cfg Normal file
View File

@ -0,0 +1,9 @@
[flake8]
max-line-length = 80
max-complexity = 15
[pep8]
max-line-length = 80
max-complexity = 15
exclude = */contrib/*
ignore = E251,E309

22
setup.py
View File

@ -67,13 +67,11 @@ setup(
"Topic :: Internet",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Internet :: Proxy Servers",
"Topic :: Software Development :: Testing"
],
"Topic :: Software Development :: Testing"],
packages=find_packages(),
include_package_data=True,
entry_points={
'console_scripts': console_scripts
},
'console_scripts': console_scripts},
install_requires=list(deps),
extras_require={
'dev': [
@ -81,19 +79,15 @@ setup(
"nose>=1.3.0",
"nose-cov>=1.6",
"coveralls>=0.4.1",
"pathod>=%s, <%s" % (
version.MINORVERSION, version.NEXT_MINORVERSION
)
],
"pathod>=%s, <%s" %
(version.MINORVERSION,
version.NEXT_MINORVERSION),
"countershape"],
'contentviews': [
"pyamf>=0.6.1",
"protobuf>=2.5.0",
"cssutils>=1.0"
],
"cssutils>=1.0"],
'examples': [
"pytz",
"harparser",
"beautifulsoup4"
]
}
)
"beautifulsoup4"]})

4
test/mock_urwid.py
View File

@ -1,4 +1,6 @@
import os, sys, mock
import os
import sys
import mock
if os.name == "nt":
m = mock.Mock()
m.__version__ = "1.1.1"

4
test/scripts/a.py
View File

@ -4,14 +4,18 @@ parser = argparse.ArgumentParser()
parser.add_argument('--var', type=int)
var = 0
def start(ctx, argv):
global var
var = parser.parse_args(argv[1:]).var
def here(ctx):
global var
var += 1
return var
def errargs():
pass

8
test/scripts/all.py
View File

@ -1,28 +1,36 @@
log = []
def clientconnect(ctx, cc):
ctx.log("XCLIENTCONNECT")
log.append("clientconnect")
def serverconnect(ctx, cc):
ctx.log("XSERVERCONNECT")
log.append("serverconnect")
def request(ctx, f):
ctx.log("XREQUEST")
log.append("request")
def response(ctx, f):
ctx.log("XRESPONSE")
log.append("response")
def responseheaders(ctx, f):
ctx.log("XRESPONSEHEADERS")
log.append("responseheaders")
def clientdisconnect(ctx, cc):
ctx.log("XCLIENTDISCONNECT")
log.append("clientdisconnect")
def error(ctx, cc):
ctx.log("XERROR")
log.append("error")

1
test/scripts/concurrent_decorator_err.py
View File

@ -1,5 +1,6 @@
from libmproxy.script import concurrent
@concurrent
def start(context, argv):
pass

1
test/scripts/duplicate_flow.py
View File

@ -2,4 +2,3 @@
def request(ctx, f):
f = ctx.duplicate_flow(f)
ctx.replay_request(f)

9
test/test_app.py
View File

@ -1,8 +1,13 @@
import mock, socket, os, time
import mock
import socket
import os
import time
from libmproxy import dump
from netlib import certutils, tcp
from libpathod.pathoc import Pathoc
import tutils, tservers
import tutils
import tservers
class TestApp(tservers.HTTPProxTest):
def test_basic(self):

29
test/test_cmdline.py
View File

@ -37,13 +37,24 @@ def test_parse_replace_hook():
def test_parse_server_spec():
tutils.raises("Invalid server specification", cmdline.parse_server_spec, "")
assert cmdline.parse_server_spec("http://foo.com:88") == [False, False, "foo.com", 88]
assert cmdline.parse_server_spec("http://foo.com") == [False, False, "foo.com", 80]
assert cmdline.parse_server_spec("https://foo.com") == [True, True, "foo.com", 443]
assert cmdline.parse_server_spec_special("https2http://foo.com") == [True, False, "foo.com", 80]
assert cmdline.parse_server_spec_special("http2https://foo.com") == [False, True, "foo.com", 443]
tutils.raises("Invalid server specification", cmdline.parse_server_spec, "foo.com")
tutils.raises("Invalid server specification", cmdline.parse_server_spec, "http://")
assert cmdline.parse_server_spec(
"http://foo.com:88") == [False, False, "foo.com", 88]
assert cmdline.parse_server_spec(
"http://foo.com") == [False, False, "foo.com", 80]
assert cmdline.parse_server_spec(
"https://foo.com") == [True, True, "foo.com", 443]
assert cmdline.parse_server_spec_special(
"https2http://foo.com") == [True, False, "foo.com", 80]
assert cmdline.parse_server_spec_special(
"http2https://foo.com") == [False, True, "foo.com", 443]
tutils.raises(
"Invalid server specification",
cmdline.parse_server_spec,
"foo.com")
tutils.raises(
"Invalid server specification",
cmdline.parse_server_spec,
"http://")
def test_parse_setheaders():
@ -103,7 +114,7 @@ def test_common():
)
p = tutils.test_data.path("data/replace")
opts.replace_file = [("/foo/bar/%s"%p)]
opts.replace_file = [("/foo/bar/%s" % p)]
v = cmdline.get_common_options(opts)["replacements"]
assert len(v) == 1
assert v[0][2].strip() == "replacecontents"
@ -122,5 +133,3 @@ def test_mitmdump():
def test_mitmweb():
ap = cmdline.mitmweb()
assert ap

6
test/test_console.py
View File

@ -1,4 +1,7 @@
import os, sys, mock, gc
import os
import sys
import mock
import gc
from os.path import normpath
import mock_urwid
from libmproxy import console
@ -6,6 +9,7 @@ from libmproxy.console import common
import tutils
class TestConsoleState:
def test_flow(self):
"""

23
test/test_console_contentview.py
View File

@ -64,7 +64,6 @@ class TestContentView:
)
assert f[0].startswith("XML")
def test_view_urlencoded(self):
d = utils.urlencode([("one", "two"), ("three", "four")])
v = cv.ViewURLEncoded()
@ -91,7 +90,7 @@ class TestContentView:
v = cv.ViewJSON()
assert v([], "{}", 1000)
assert not v([], "{", 1000)
assert v([], "[" + ",".join(["0"]*cv.VIEW_CUTOFF) + "]", 1000)
assert v([], "[" + ",".join(["0"] * cv.VIEW_CUTOFF) + "]", 1000)
assert v([], "[1, 2, 3, 4, 5]", 5)
def test_view_xml(self):
@ -145,18 +144,18 @@ class TestContentView:
def test_view_image(self):
v = cv.ViewImage()
p = tutils.test_data.path("data/image.png")
assert v([], file(p,"rb").read(), sys.maxint)
assert v([], file(p, "rb").read(), sys.maxsize)
p = tutils.test_data.path("data/image.gif")
assert v([], file(p,"rb").read(), sys.maxint)
assert v([], file(p, "rb").read(), sys.maxsize)
p = tutils.test_data.path("data/image-err1.jpg")
assert v([], file(p,"rb").read(), sys.maxint)
assert v([], file(p, "rb").read(), sys.maxsize)
p = tutils.test_data.path("data/image.ico")
assert v([], file(p,"rb").read(), sys.maxint)
assert v([], file(p, "rb").read(), sys.maxsize)
assert not v([], "flibble", sys.maxint)
assert not v([], "flibble", sys.maxsize)
def test_view_multipart(self):
view = cv.ViewMultipart()
@ -226,7 +225,6 @@ Larry
)
assert "Raw" in r[0]
r = cv.get_content_view(
cv.get("Auto"),
[
@ -261,24 +259,25 @@ if pyamf:
v = cv.ViewAMF()
p = tutils.test_data.path("data/amf01")
assert v([], file(p,"rb").read(), sys.maxint)
assert v([], file(p, "rb").read(), sys.maxsize)
p = tutils.test_data.path("data/amf02")
assert v([], file(p,"rb").read(), sys.maxint)
assert v([], file(p, "rb").read(), sys.maxsize)
def test_view_amf_response():
v = cv.ViewAMF()
p = tutils.test_data.path("data/amf03")
assert v([], file(p,"rb").read(), sys.maxint)
assert v([], file(p, "rb").read(), sys.maxsize)
if cv.ViewProtobuf.is_available():
def test_view_protobuf_request():
v = cv.ViewProtobuf()
p = tutils.test_data.path("data/protobuf01")
content_type, output = v([], file(p,"rb").read(), sys.maxint)
content_type, output = v([], file(p, "rb").read(), sys.maxsize)
assert content_type == "Protobuf"
assert output[0].text == '1: "3bbc333c-e61c-433b-819a-0b9a8cc103b8"'
def test_get_by_shortcut():
assert cv.get_by_shortcut("h")

2
test/test_console_help.py
View File

@ -5,10 +5,12 @@ if os.name == "nt":
import libmproxy.console.help as help
class DummyLoop:
def __init__(self):
self.widget = None
class DummyMaster:
def __init__(self):
self.loop = DummyLoop()

2
test/test_controller.py
View File

@ -8,5 +8,3 @@ class TestMaster:
msg = mock.MagicMock()
m.handle("type", msg)
assert msg.reply.call_count == 1

17
test/test_dump.py
View File

@ -116,7 +116,6 @@ class TestDumpMaster:
0, None, "", verbosity=1, rfile="test_dump.py"
)
def test_options(self):
o = dump.Options(verbosity = 2)
assert o.verbosity == 2
@ -147,21 +146,25 @@ class TestDumpMaster:
def test_basic(self):
for i in (1, 2, 3):
assert "GET" in self._dummy_cycle(1, "~s", "", flow_detail=i)
assert "GET" in self._dummy_cycle(1, "~s", "\x00\x00\x00", flow_detail=i)
assert "GET" in self._dummy_cycle(
1,
"~s",
"\x00\x00\x00",
flow_detail=i)
assert "GET" in self._dummy_cycle(1, "~s", "ascii", flow_detail=i)
def test_write(self):
with tutils.tmpdir() as d:
p = os.path.join(d, "a")
self._dummy_cycle(1, None, "", outfile=(p,"wb"), verbosity=0)
assert len(list(flow.FlowReader(open(p,"rb")).stream())) == 1
self._dummy_cycle(1, None, "", outfile=(p, "wb"), verbosity=0)
assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 1
def test_write_append(self):
with tutils.tmpdir() as d:
p = os.path.join(d, "a.append")
self._dummy_cycle(1, None, "", outfile=(p,"wb"), verbosity=0)
self._dummy_cycle(1, None, "", outfile=(p,"ab"), verbosity=0)
assert len(list(flow.FlowReader(open(p,"rb")).stream())) == 2
self._dummy_cycle(1, None, "", outfile=(p, "wb"), verbosity=0)
self._dummy_cycle(1, None, "", outfile=(p, "ab"), verbosity=0)
assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 2
def test_write_err(self):
tutils.raises(

22
test/test_encoding.py
View File

@ -1,5 +1,6 @@
from libmproxy import encoding
def test_identity():
assert "string" == encoding.decode("identity", "string")
assert "string" == encoding.encode("identity", "string")
@ -8,12 +9,25 @@ def test_identity():
def test_gzip():
assert "string" == encoding.decode("gzip", encoding.encode("gzip", "string"))
assert "string" == encoding.decode(
"gzip",
encoding.encode(
"gzip",
"string"))
assert None == encoding.decode("gzip", "bogus")
def test_deflate():
assert "string" == encoding.decode("deflate", encoding.encode("deflate", "string"))
assert "string" == encoding.decode("deflate", encoding.encode("deflate", "string")[2:-4])
assert "string" == encoding.decode(
"deflate",
encoding.encode(
"deflate",
"string"))
assert "string" == encoding.decode(
"deflate",
encoding.encode(
"deflate",
"string")[
2:-
4])
assert None == encoding.decode("deflate", "bogus")

2
test/test_examples.py
View File

@ -21,7 +21,7 @@ def test_load_scripts():
f += " foo bar" # two arguments required
try:
s = script.Script(f, tmaster) # Loads the script file.
except Exception, v:
except Exception as v:
if not "ImportError" in str(v):
raise
else:

11
test/test_filt.py
View File

@ -5,6 +5,7 @@ from libmproxy.protocol import http
from libmproxy.protocol.primitives import Error
import tutils
class TestParsing:
def _dump(self, x):
c = cStringIO.StringIO()
@ -99,7 +100,15 @@ class TestMatching:
headers = odict.ODictCaseless()
headers["header_response"] = ["svalue"]
f.response = http.HTTPResponse((1, 1), 200, "OK", headers, "content_response", None, None)
f.response = http.HTTPResponse(
(1,
1),
200,
"OK",
headers,
"content_response",
None,
None)
return f

221
test/test_flow.py
View File

@ -1,4 +1,6 @@
import Queue, time, os.path
import Queue
import time
import os.path
from cStringIO import StringIO
import email.utils
import mock
@ -33,7 +35,6 @@ def test_app_registry():
assert ar.get(r)
class TestStickyCookieState:
def _response(self, cookie, host):
s = flow.StickyCookieState(filt.parse(".*"))
@ -115,7 +116,15 @@ class TestClientPlaybackState:
class TestServerPlaybackState:
def test_hash(self):
s = flow.ServerPlaybackState(None, [], False, False, None, False, None, False)
s = flow.ServerPlaybackState(
None,
[],
False,
False,
None,
False,
None,
False)
r = tutils.tflow()
r2 = tutils.tflow()
@ -131,7 +140,15 @@ class TestServerPlaybackState:
assert s._hash(r) != s._hash(r2)
def test_headers(self):
s = flow.ServerPlaybackState(["foo"], [], False, False, None, False, None, False)
s = flow.ServerPlaybackState(
["foo"],
[],
False,
False,
None,
False,
None,
False)
r = tutils.tflow(resp=True)
r.request.headers["foo"] = ["bar"]
r2 = tutils.tflow(resp=True)
@ -152,7 +169,9 @@ class TestServerPlaybackState:
r2 = tutils.tflow(resp=True)
r2.request.headers["key"] = ["two"]
s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False, None, False)
s = flow.ServerPlaybackState(
None, [
r, r2], False, False, None, False, None, False)
assert s.count() == 2
assert len(s.fmap.keys()) == 1
@ -173,34 +192,41 @@ class TestServerPlaybackState:
r2 = tutils.tflow(resp=True)
r2.request.headers["key"] = ["two"]
s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False, None, False)
s = flow.ServerPlaybackState(
None, [
r, r2], False, True, None, False, None, False)
assert s.count() == 2
s.next_flow(r)
assert s.count() == 2
def test_ignore_params(self):
s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False, None, False)
s = flow.ServerPlaybackState(
None, [], False, False, [
"param1", "param2"], False, None, False)
r = tutils.tflow(resp=True)
r.request.path="/test?param1=1"
r.request.path = "/test?param1=1"
r2 = tutils.tflow(resp=True)
r2.request.path="/test"
r2.request.path = "/test"
assert s._hash(r) == s._hash(r2)
r2.request.path="/test?param1=2"
r2.request.path = "/test?param1=2"
assert s._hash(r) == s._hash(r2)
r2.request.path="/test?param2=1"
r2.request.path = "/test?param2=1"
assert s._hash(r) == s._hash(r2)
r2.request.path="/test?param3=2"
r2.request.path = "/test?param3=2"
assert not s._hash(r) == s._hash(r2)
def test_ignore_payload_params(self):
s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"], False)
s = flow.ServerPlaybackState(
None, [], False, False, None, False, [
"param1", "param2"], False)
r = tutils.tflow(resp=True)
r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"]
r.request.headers[
"Content-Type"] = ["application/x-www-form-urlencoded"]
r.request.content = "paramx=x&param1=1"
r2 = tutils.tflow(resp=True)
r2.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"]
r2.request.headers[
"Content-Type"] = ["application/x-www-form-urlencoded"]
r2.request.content = "paramx=x&param1=1"
# same parameters
assert s._hash(r) == s._hash(r2)
@ -208,20 +234,22 @@ class TestServerPlaybackState:
r2.request.content = "paramx=x&param1=2"
assert s._hash(r) == s._hash(r2)
# missing parameter
r2.request.content="paramx=x"
r2.request.content = "paramx=x"
assert s._hash(r) == s._hash(r2)
# ignorable parameter added
r2.request.content="paramx=x&param1=2"
r2.request.content = "paramx=x&param1=2"
assert s._hash(r) == s._hash(r2)
# not ignorable parameter changed
r2.request.content="paramx=y&param1=1"
r2.request.content = "paramx=y&param1=1"
assert not s._hash(r) == s._hash(r2)
# not ignorable parameter missing
r2.request.content="param1=1"
r2.request.content = "param1=1"
assert not s._hash(r) == s._hash(r2)
def test_ignore_payload_params_other_content_type(self):
s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"], False)
s = flow.ServerPlaybackState(
None, [], False, False, None, False, [
"param1", "param2"], False)
r = tutils.tflow(resp=True)
r.request.headers["Content-Type"] = ["application/json"]
r.request.content = '{"param1":"1"}'
@ -235,19 +263,31 @@ class TestServerPlaybackState:
assert not s._hash(r) == s._hash(r2)
def test_ignore_payload_wins_over_params(self):
#NOTE: parameters are mutually exclusive in options
s = flow.ServerPlaybackState(None, [], False, False, None, True, ["param1", "param2"], False)
# NOTE: parameters are mutually exclusive in options
s = flow.ServerPlaybackState(
None, [], False, False, None, True, [
"param1", "param2"], False)
r = tutils.tflow(resp=True)
r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"]
r.request.headers[
"Content-Type"] = ["application/x-www-form-urlencoded"]
r.request.content = "paramx=y"
r2 = tutils.tflow(resp=True)
r2.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"]
r2.request.headers[
"Content-Type"] = ["application/x-www-form-urlencoded"]
r2.request.content = "paramx=x"
# same parameters
assert s._hash(r) == s._hash(r2)
def test_ignore_content(self):
s = flow.ServerPlaybackState(None, [], False, False, None, False, None, False)
s = flow.ServerPlaybackState(
None,
[],
False,
False,
None,
False,
None,
False)
r = tutils.tflow(resp=True)
r2 = tutils.tflow(resp=True)
@ -257,8 +297,16 @@ class TestServerPlaybackState:
r2.request.content = "bar"
assert not s._hash(r) == s._hash(r2)
#now ignoring content
s = flow.ServerPlaybackState(None, [], False, False, None, True, None, False)
# now ignoring content
s = flow.ServerPlaybackState(
None,
[],
False,
False,
None,
True,
None,
False)
r = tutils.tflow(resp=True)
r2 = tutils.tflow(resp=True)
r.request.content = "foo"
@ -272,14 +320,22 @@ class TestServerPlaybackState:
assert s._hash(r) == s._hash(r2)
def test_ignore_host(self):
s = flow.ServerPlaybackState(None, [], False, False, None, False, None, True)
s = flow.ServerPlaybackState(
None,
[],
False,
False,
None,
False,
None,
True)
r = tutils.tflow(resp=True)
r2 = tutils.tflow(resp=True)
r.request.host="address"
r2.request.host="address"
r.request.host = "address"
r2.request.host = "address"
assert s._hash(r) == s._hash(r2)
r2.request.host="wrong_address"
r2.request.host = "wrong_address"
assert s._hash(r) == s._hash(r2)
@ -343,12 +399,14 @@ class TestFlow:
def test_getset_state(self):
f = tutils.tflow(resp=True)
state = f.get_state()
assert f.get_state() == protocol.http.HTTPFlow.from_state(state).get_state()
assert f.get_state() == protocol.http.HTTPFlow.from_state(
state).get_state()
f.response = None
f.error = Error("error")
state = f.get_state()
assert f.get_state() == protocol.http.HTTPFlow.from_state(state).get_state()
assert f.get_state() == protocol.http.HTTPFlow.from_state(
state).get_state()
f2 = f.copy()
f2.id = f.id # copy creates a different uuid
@ -430,7 +488,6 @@ class TestFlow:
assert f.response.content == "abarb"
class TestState:
def test_backup(self):
c = flow.State()
@ -519,7 +576,7 @@ class TestState:
assert c.intercept_txt == "~q"
assert "Invalid" in c.set_intercept("~")
assert not c.set_intercept(None)
assert c.intercept_txt == None
assert c.intercept_txt is None
def _add_request(self, state):
f = tutils.tflow()
@ -608,7 +665,13 @@ class TestSerialize:
def test_load_flows_reverse(self):
r = self._treader()
s = flow.State()
conf = ProxyConfig(mode="reverse", upstream_server=[True,True,"use-this-domain",80])
conf = ProxyConfig(
mode="reverse",
upstream_server=[
True,
True,
"use-this-domain",
80])
fm = flow.FlowMaster(DummyServer(conf), s)
fm.load_flows(r)
assert s.flows[0].request.host == "use-this-domain"
@ -630,7 +693,6 @@ class TestSerialize:
r = flow.FlowReader(sio)
assert len(list(r.stream()))
def test_error(self):
sio = StringIO()
sio.write("bogus")
@ -661,7 +723,8 @@ class TestFlowMaster:
assert not fm.load_script(tutils.test_data.path("scripts/a.py"))
assert not fm.unload_scripts()
assert fm.load_script("nonexistent")
assert "ValueError" in fm.load_script(tutils.test_data.path("scripts/starterr.py"))
assert "ValueError" in fm.load_script(
tutils.test_data.path("scripts/starterr.py"))
assert len(fm.scripts) == 0
def test_getset_ignore(self):
@ -707,14 +770,14 @@ class TestFlowMaster:
assert fm.scripts[0].ns["log"][-1] == "request"
fm.handle_response(f)
assert fm.scripts[0].ns["log"][-1] == "response"
#load second script
# load second script
assert not fm.load_script(tutils.test_data.path("scripts/all.py"))
assert len(fm.scripts) == 2
fm.handle_clientdisconnect(f.server_conn)
assert fm.scripts[0].ns["log"][-1] == "clientdisconnect"
assert fm.scripts[1].ns["log"][-1] == "clientdisconnect"
#unload first script
# unload first script
fm.unload_scripts()
assert len(fm.scripts) == 0
assert not fm.load_script(tutils.test_data.path("scripts/all.py"))
@ -765,7 +828,16 @@ class TestFlowMaster:
f = tutils.tflow(resp=True)
pb = [tutils.tflow(resp=True), f]
fm = flow.FlowMaster(DummyServer(ProxyConfig()), s)
assert not fm.start_server_playback(pb, False, [], False, False, None, False, None, False)
assert not fm.start_server_playback(
pb,
False,
[],
False,
False,
None,
False,
None,
False)
assert not fm.start_client_playback(pb, False)
fm.client_playback.testing = True
@ -788,16 +860,43 @@ class TestFlowMaster:
fm.refresh_server_playback = True
assert not fm.do_server_playback(tutils.tflow())
fm.start_server_playback(pb, False, [], False, False, None, False, None, False)
fm.start_server_playback(
pb,
False,
[],
False,
False,
None,
False,
None,
False)
assert fm.do_server_playback(tutils.tflow())
fm.start_server_playback(pb, False, [], True, False, None, False, None, False)
fm.start_server_playback(
pb,
False,
[],
True,
False,
None,
False,
None,
False)
r = tutils.tflow()
r.request.content = "gibble"
assert not fm.do_server_playback(r)
assert fm.do_server_playback(tutils.tflow())
fm.start_server_playback(pb, False, [], True, False, None, False, None, False)
fm.start_server_playback(
pb,
False,
[],
True,
False,
None,
False,
None,
False)
q = Queue.Queue()
fm.tick(q, 0)
assert fm.should_exit.is_set()
@ -812,7 +911,16 @@ class TestFlowMaster:
pb = [f]
fm = flow.FlowMaster(None, s)
fm.refresh_server_playback = True
fm.start_server_playback(pb, True, [], False, False, None, False, None, False)
fm.start_server_playback(
pb,
True,
[],
False,
False,
None,
False,
None,
False)
f = tutils.tflow()
f.request.host = "nonexistent"
@ -862,8 +970,9 @@ class TestFlowMaster:
def test_stream(self):
with tutils.tmpdir() as tdir:
p = os.path.join(tdir, "foo")
def r():
r = flow.FlowReader(open(p,"rb"))
r = flow.FlowReader(open(p, "rb"))
return list(r.stream())
s = flow.State()
@ -884,6 +993,7 @@ class TestFlowMaster:
assert not r()[1].response
class TestRequest:
def test_simple(self):
f = tutils.tflow()
@ -919,7 +1029,7 @@ class TestRequest:
r.host = "address"
r.port = 22
assert r.url== "https://address:22/path"
assert r.url == "https://address:22/path"
assert r.pretty_url(True) == "https://address:22/path"
r.headers["Host"] = ["foo.com"]
@ -1062,6 +1172,7 @@ class TestRequest:
resp.headers = h
assert resp.headers.get_first("content-type") == "text/plain"
class TestResponse:
def test_simple(self):
f = tutils.tflow(resp=True)
@ -1077,7 +1188,9 @@ class TestResponse:
assert resp.size() == len(resp.assemble())
resp.content = CONTENT_MISSING
tutils.raises("Cannot assemble flow with CONTENT_MISSING", resp.assemble)
tutils.raises(
"Cannot assemble flow with CONTENT_MISSING",
resp.assemble)
def test_refresh(self):
r = tutils.tresp()
@ -1086,14 +1199,15 @@ class TestResponse:
pre = r.headers["date"]
r.refresh(n)
assert pre == r.headers["date"]
r.refresh(n+60)
r.refresh(n + 60)
d = email.utils.parsedate_tz(r.headers["date"][0])
d = email.utils.mktime_tz(d)
# Weird that this is not exact...
assert abs(60-(d-n)) <= 1
assert abs(60 - (d - n)) <= 1
r.headers["set-cookie"] = ["MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure"]
r.headers[
"set-cookie"] = ["MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure"]
r.refresh()
def test_refresh_cookie(self):
@ -1146,7 +1260,7 @@ class TestResponse:
def test_header_size(self):
r = tutils.tresp()
result = len(r._assemble_headers())
assert result==44
assert result == 44
def test_get_content_type(self):
h = odict.ODictCaseless()
@ -1295,7 +1409,6 @@ def test_setheaders():
h.run(f)
assert f.request.content == "foo"
h.clear()
h.add("~s", "one", "two")
h.add("~s", "one", "three")

9
test/test_fuzzing.py
View File

@ -5,16 +5,17 @@ import tservers
after being fixed to check for regressions.
"""
class TestFuzzy(tservers.HTTPProxTest):
def test_idna_err(self):
req = r'get:"http://localhost:%s":i10,"\xc6"'
p = self.pathoc()
assert p.request(req%self.server.port).status_code == 400
assert p.request(req % self.server.port).status_code == 400
def test_nullbytes(self):
req = r'get:"http://localhost:%s":i19,"\x00"'
p = self.pathoc()
assert p.request(req%self.server.port).status_code == 400
assert p.request(req % self.server.port).status_code == 400
def test_invalid_ports(self):
req = 'get:"http://localhost:999999"'
@ -24,12 +25,12 @@ class TestFuzzy(tservers.HTTPProxTest):
def test_invalid_ipv6_url(self):
req = 'get:"http://localhost:%s":i13,"["'
p = self.pathoc()
assert p.request(req%self.server.port).status_code == 400
assert p.request(req % self.server.port).status_code == 400
def test_invalid_upstream(self):
req = r"get:'http://localhost:%s/p/200:i10,\x27+\x27'"
p = self.pathoc()
assert p.request(req%self.server.port).status_code == 502
assert p.request(req % self.server.port).status_code == 502
def test_upstream_disconnect(self):
req = r'200:d0'

21
test/test_platform_pf.py
View File

@ -1,4 +1,5 @@
import tutils, sys
import tutils
import sys
from libmproxy.platform import pf
@ -6,10 +7,20 @@ class TestLookup:
def test_simple(self):
if sys.platform == "freebsd10":
p = tutils.test_data.path("data/pf02")
d = open(p,"rb").read()
d = open(p, "rb").read()
else:
p = tutils.test_data.path("data/pf01")
d = open(p,"rb").read()
d = open(p, "rb").read()
assert pf.lookup("192.168.1.111", 40000, d) == ("5.5.5.5", 80)
tutils.raises("Could not resolve original destination", pf.lookup, "192.168.1.112", 40000, d)
tutils.raises("Could not resolve original destination", pf.lookup, "192.168.1.111", 40001, d)
tutils.raises(
"Could not resolve original destination",
pf.lookup,
"192.168.1.112",
40000,
d)
tutils.raises(
"Could not resolve original destination",
pf.lookup,
"192.168.1.111",
40001,
d)

13
test/test_protocol_http.py
View File

@ -61,7 +61,8 @@ class TestHTTPRequest:
assert "Host" in r.headers
def test_expect_header(self):
s = StringIO("GET / HTTP/1.1\r\nContent-Length: 3\r\nExpect: 100-continue\r\n\r\nfoobar")
s = StringIO(
"GET / HTTP/1.1\r\nContent-Length: 3\r\nExpect: 100-continue\r\n\r\nfoobar")
w = StringIO()
r = HTTPRequest.from_stream(s, wfile=w)
assert w.getvalue() == "HTTP/1.1 100 Continue\r\n\r\n"
@ -84,7 +85,8 @@ class TestHTTPRequest:
tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s)
s = StringIO("GET http://address:22/ HTTP/1.1")
r = HTTPRequest.from_stream(s)
assert r.assemble() == "GET http://address:22/ HTTP/1.1\r\nHost: address:22\r\nContent-Length: 0\r\n\r\n"
assert r.assemble(
) == "GET http://address:22/ HTTP/1.1\r\nHost: address:22\r\nContent-Length: 0\r\n\r\n"
def test_http_options_relative_form_in(self):
"""
@ -105,11 +107,11 @@ class TestHTTPRequest:
r.host = 'address'
r.port = 80
r.scheme = "http"
assert r.assemble() == ("OPTIONS http://address:80/secret/resource HTTP/1.1\r\n"
assert r.assemble() == (
"OPTIONS http://address:80/secret/resource HTTP/1.1\r\n"
"Host: address\r\n"
"Content-Length: 0\r\n\r\n")
def test_assemble_unknown_form(self):
r = tutils.treq()
tutils.raises("Invalid request form", r.assemble, "antiauthority")
@ -257,7 +259,8 @@ class TestHTTPResponse:
def test_get_cookies_with_parameters(self):
h = odict.ODictCaseless()
h["Set-Cookie"] = ["cookiename=cookievalue;domain=example.com;expires=Wed Oct 21 16:29:41 2015;path=/; HttpOnly"]
h["Set-Cookie"] = [
"cookiename=cookievalue;domain=example.com;expires=Wed Oct 21 16:29:41 2015;path=/; HttpOnly"]
resp = tutils.tresp()
resp.headers = h
result = resp.get_cookies()

35
test/test_proxy.py
View File

@ -78,7 +78,6 @@ class TestProcessProxyOptions:
def test_no_transparent(self):
self.assert_err("transparent mode not supported", "-T")
@mock.patch("libmproxy.platform.resolver")
def test_modes(self, _):
self.assert_noerr("-R", "http://localhost")
@ -96,28 +95,42 @@ class TestProcessProxyOptions:
def test_client_certs(self):
with tutils.tmpdir() as cadir:
self.assert_noerr("--client-certs", cadir)
self.assert_err("directory does not exist", "--client-certs", "nonexistent")
self.assert_err(
"directory does not exist",
"--client-certs",
"nonexistent")
def test_certs(self):
with tutils.tmpdir() as cadir:
self.assert_noerr("--cert", tutils.test_data.path("data/testkey.pem"))
self.assert_noerr(
"--cert",
tutils.test_data.path("data/testkey.pem"))
self.assert_err("does not exist", "--cert", "nonexistent")
def test_auth(self):
p = self.assert_noerr("--nonanonymous")
assert p.authenticator
p = self.assert_noerr("--htpasswd", tutils.test_data.path("data/htpasswd"))
p = self.assert_noerr(
"--htpasswd",
tutils.test_data.path("data/htpasswd"))
assert p.authenticator
self.assert_err("malformed htpasswd file", "--htpasswd", tutils.test_data.path("data/htpasswd.invalid"))
self.assert_err(
"malformed htpasswd file",
"--htpasswd",
tutils.test_data.path("data/htpasswd.invalid"))
p = self.assert_noerr("--singleuser", "test:test")
assert p.authenticator
self.assert_err("invalid single-user specification", "--singleuser", "test")
self.assert_err(
"invalid single-user specification",
"--singleuser",
"test")
class TestProxyServer:
@tutils.SkipWindows # binding to 0.0.0.0:1 works without special permissions on Windows
# binding to 0.0.0.0:1 works without special permissions on Windows
@tutils.SkipWindows
def test_err(self):
conf = ProxyConfig(
port=1
@ -142,6 +155,12 @@ class TestConnectionHandler:
def test_fatal_error(self):
config = mock.Mock()
config.mode.get_upstream_server.side_effect = RuntimeError
c = ConnectionHandler(config, mock.MagicMock(), ("127.0.0.1", 8080), None, mock.MagicMock())
c = ConnectionHandler(
config,
mock.MagicMock(),
("127.0.0.1",
8080),
None,
mock.MagicMock())
with tutils.capture_stderr(c.handle) as output:
assert "mitmproxy has crashed" in output

13
test/test_script.py
View File

@ -11,7 +11,7 @@ class TestScript:
s = flow.State()
fm = flow.FlowMaster(None, s)
sp = tutils.test_data.path("scripts/a.py")
p = script.Script("%s --var 40"%sp, fm)
p = script.Script("%s --var 40" % sp, fm)
assert "here" in p.ns
assert p.run("here") == (True, 41)
@ -79,7 +79,9 @@ class TestScript:
def test_concurrent2(self):
s = flow.State()
fm = flow.FlowMaster(None, s)
s = script.Script(tutils.test_data.path("scripts/concurrent_decorator.py"), fm)
s = script.Script(
tutils.test_data.path("scripts/concurrent_decorator.py"),
fm)
s.load()
m = mock.Mock()
@ -110,8 +112,9 @@ class TestScript:
fm = flow.FlowMaster(None, s)
tutils.raises(
"decorator not supported for this method",
script.Script, tutils.test_data.path("scripts/concurrent_decorator_err.py"), fm
)
script.Script,
tutils.test_data.path("scripts/concurrent_decorator_err.py"),
fm)
def test_command_parsing():
@ -120,5 +123,3 @@ def test_command_parsing():
absfilepath = os.path.normcase(tutils.test_data.path("scripts/a.py"))
s = script.Script(absfilepath, fm)
assert os.path.isfile(s.argv[0])

216
test/test_server.py
View File

@ -1,10 +1,12 @@
import socket, time
import socket
import time
from libmproxy.proxy.config import HostMatcher
import libpathod
from netlib import tcp, http_auth, http
from libpathod import pathoc, pathod
from netlib.certutils import SSLCert
import tutils, tservers
import tutils
import tservers
from libmproxy.protocol import KILL, Error
from libmproxy.protocol.http import CONTENT_MISSING
@ -16,9 +18,10 @@ from libmproxy.protocol.http import CONTENT_MISSING
for a 200 response.
"""
class CommonMixin:
def test_large(self):
assert len(self.pathod("200:b@50k").content) == 1024*50
assert len(self.pathod("200:b@50k").content) == 1024 * 50
@staticmethod
def wait_until_not_live(flow):
@ -56,7 +59,8 @@ class CommonMixin:
# Port error
l.request.port = 1
# In upstream mode, we get a 502 response from the upstream proxy server.
# In upstream mode with ssl, the replay will fail as we cannot establish SSL with the upstream proxy.
# In upstream mode with ssl, the replay will fail as we cannot establish
# SSL with the upstream proxy.
rt = self.master.replay_request(l, block=True)
assert not rt
if isinstance(self, tservers.HTTPUpstreamProxTest) and not self.ssl:
@ -68,7 +72,9 @@ class CommonMixin:
f = self.pathod("304")
assert f.status_code == 304
l = self.master.state.view[-1] # In Upstream mode with SSL, we may already have a previous CONNECT request.
# In Upstream mode with SSL, we may already have a previous CONNECT
# request.
l = self.master.state.view[-1]
assert l.client_conn.address
assert "host" in l.request.headers
assert l.response.code == 304
@ -90,11 +96,13 @@ class CommonMixin:
log = self.server.last_log()
assert log["request"]["sni"] == "testserver.com"
class TcpMixin:
def _ignore_on(self):
assert not hasattr(self, "_ignore_backup")
self._ignore_backup = self.config.check_ignore
self.config.check_ignore = HostMatcher([".+:%s" % self.server.port] + self.config.check_ignore.patterns)
self.config.check_ignore = HostMatcher(
[".+:%s" % self.server.port] + self.config.check_ignore.patterns)
def _ignore_off(self):
assert hasattr(self, "_ignore_backup")
@ -125,22 +133,26 @@ class TcpMixin:
# Test Non-HTTP traffic
spec = "200:i0,@100:d0" # this results in just 100 random bytes
assert self.pathod(spec).status_code == 502 # mitmproxy responds with bad gateway
# mitmproxy responds with bad gateway
assert self.pathod(spec).status_code == 502
self._ignore_on()
tutils.raises("invalid server response", self.pathod, spec) # pathoc tries to parse answer as HTTP
tutils.raises(
"invalid server response",
self.pathod,
spec) # pathoc tries to parse answer as HTTP
self._ignore_off()
def _tcpproxy_on(self):
assert not hasattr(self, "_tcpproxy_backup")
self._tcpproxy_backup = self.config.check_tcp
self.config.check_tcp = HostMatcher([".+:%s" % self.server.port] + self.config.check_tcp.patterns)
self.config.check_tcp = HostMatcher(
[".+:%s" % self.server.port] + self.config.check_tcp.patterns)
def _tcpproxy_off(self):
assert hasattr(self, "_tcpproxy_backup")
self.config.check_ignore = self._tcpproxy_backup
del self._tcpproxy_backup
def test_tcp(self):
spec = '304:h"Alternate-Protocol"="mitmproxy-will-remove-this"'
n = self.pathod(spec)
@ -165,6 +177,7 @@ class TcpMixin:
# Make sure that TCP messages are in the event log.
assert any("mitmproxy-will-remove-this" in m for m in self.master.log)
class AppMixin:
def test_app(self):
ret = self.app("/")
@ -188,30 +201,30 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin):
def test_upstream_ssl_error(self):
p = self.pathoc()
ret = p.request("get:'https://localhost:%s/'"%self.server.port)
ret = p.request("get:'https://localhost:%s/'" % self.server.port)
assert ret.status_code == 400
def test_connection_close(self):
# Add a body, so we have a content-length header, which combined with
# HTTP1.1 means the connection is kept alive.
response = '%s/p/200:b@1'%self.server.urlbase
response = '%s/p/200:b@1' % self.server.urlbase
# Lets sanity check that the connection does indeed stay open by
# issuing two requests over the same connection
p = self.pathoc()
assert p.request("get:'%s'"%response)
assert p.request("get:'%s'"%response)
assert p.request("get:'%s'" % response)
assert p.request("get:'%s'" % response)
# Now check that the connection is closed as the client specifies
p = self.pathoc()
assert p.request("get:'%s':h'Connection'='close'"%response)
assert p.request("get:'%s':h'Connection'='close'" % response)
# There's a race here, which means we can get any of a number of errors.
# Rather than introduce yet another sleep into the test suite, we just
# relax the Exception specification.
tutils.raises(Exception, p.request, "get:'%s'"%response)
tutils.raises(Exception, p.request, "get:'%s'" % response)
def test_reconnect(self):
req = "get:'%s/p/200:b@1:da'"%self.server.urlbase
req = "get:'%s/p/200:b@1:da'" % self.server.urlbase
p = self.pathoc()
assert p.request(req)
# Server has disconnected. Mitmproxy should detect this, and reconnect.
@ -225,8 +238,8 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin):
return True
req = "get:'%s/p/200:b@1'"
p = self.pathoc()
assert p.request(req%self.server.urlbase)
assert p.request(req%self.server2.urlbase)
assert p.request(req % self.server.urlbase)
assert p.request(req % self.server2.urlbase)
assert switched(self.proxy.log)
def test_get_connection_err(self):
@ -237,7 +250,7 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin):
def test_blank_leading_line(self):
p = self.pathoc()
req = "get:'%s/p/201':i0,'\r\n'"
assert p.request(req%self.server.urlbase).status_code == 201
assert p.request(req % self.server.urlbase).status_code == 201
def test_invalid_headers(self):
p = self.pathoc()
@ -251,7 +264,9 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin):
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connection.connect(("127.0.0.1", self.proxy.port))
spec = '301:h"Transfer-Encoding"="chunked":r:b"0\\r\\n\\r\\n"'
connection.send("GET http://localhost:%d/p/%s HTTP/1.1\r\n"%(self.server.port, spec))
connection.send(
"GET http://localhost:%d/p/%s HTTP/1.1\r\n" %
(self.server.port, spec))
connection.send("\r\n")
resp = connection.recv(50000)
connection.close()
@ -270,13 +285,20 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin):
self.master.set_stream_large_bodies(None)
def test_stream_modify(self):
self.master.load_script(tutils.test_data.path("scripts/stream_modify.py"))
self.master.load_script(
tutils.test_data.path("scripts/stream_modify.py"))
d = self.pathod('200:b"foo"')
assert d.content == "bar"
self.master.unload_scripts()
class TestHTTPAuth(tservers.HTTPProxTest):
authenticator = http_auth.BasicProxyAuth(http_auth.PassManSingleUser("test", "test"), "realm")
authenticator = http_auth.BasicProxyAuth(
http_auth.PassManSingleUser(
"test",
"test"),
"realm")
def test_auth(self):
assert self.pathod("202").status_code == 407
p = self.pathoc()
@ -284,7 +306,7 @@ class TestHTTPAuth(tservers.HTTPProxTest):
get
'http://localhost:%s/p/202'
h'%s'='%s'
"""%(
""" % (
self.server.port,
http_auth.BasicProxyAuth.AUTH_HEADER,
http.assemble_http_basic_auth("basic", "test", "test")
@ -294,6 +316,7 @@ class TestHTTPAuth(tservers.HTTPProxTest):
class TestHTTPConnectSSLError(tservers.HTTPProxTest):
certfile = True
def test_go(self):
self.config.ssl_ports.append(self.proxy.port)
p = self.pathoc_raw()
@ -306,6 +329,7 @@ class TestHTTPS(tservers.HTTPProxTest, CommonMixin, TcpMixin):
ssl = True
ssloptions = pathod.SSLOptions(request_client_cert=True)
clientcerts = True
def test_clientcert(self):
f = self.pathod("304")
assert f.status_code == 304
@ -319,6 +343,7 @@ class TestHTTPS(tservers.HTTPProxTest, CommonMixin, TcpMixin):
class TestHTTPSCertfile(tservers.HTTPProxTest, CommonMixin):
ssl = True
certfile = True
def test_certfile(self):
assert self.pathod("304")
@ -328,11 +353,12 @@ class TestHTTPSNoCommonName(tservers.HTTPProxTest):
Test what happens if we get a cert without common name back.
"""
ssl = True
ssloptions=pathod.SSLOptions(
ssloptions = pathod.SSLOptions(
certs = [
("*", tutils.test_data.path("data/no_common_name.pem"))
]
)
def test_http(self):
f = self.pathod("202")
assert f.sslinfo.certchain[0].get_subject().CN == "127.0.0.1"
@ -373,7 +399,6 @@ class TestHttps2Http(tservers.ReverseProxTest):
assert p.request("get:'/p/200'").status_code == 400
class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin):
ssl = False
@ -413,15 +438,19 @@ class TestProxy(tservers.HTTPProxTest):
connection.connect(("127.0.0.1", self.proxy.port))
# call pathod server, wait a second to complete the request
connection.send("GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n"%self.server.port)
connection.send(
"GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n" %
self.server.port)
time.sleep(1)
connection.send("\r\n")
connection.recv(50000)
connection.close()
request, response = self.master.state.view[0].request, self.master.state.view[0].response
request, response = self.master.state.view[
0].request, self.master.state.view[0].response
assert response.code == 304 # sanity test for our low level request
assert 0.95 < (request.timestamp_end - request.timestamp_start) < 1.2 #time.sleep might be a little bit shorter than a second
# time.sleep might be a little bit shorter than a second
assert 0.95 < (request.timestamp_end - request.timestamp_start) < 1.2
def test_request_timestamps_not_affected_by_client_time(self):
# test that don't include user wait time in request's timestamps
@ -441,10 +470,14 @@ class TestProxy(tservers.HTTPProxTest):
# tests that the client_conn a tcp connection has a tcp_setup_timestamp
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connection.connect(("localhost", self.proxy.port))
connection.send("GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n"%self.server.port)
connection.send(
"GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n" %
self.server.port)
connection.send("\r\n")
connection.recv(5000)
connection.send("GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n"%self.server.port)
connection.send(
"GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n" %
self.server.port)
connection.send("\r\n")
connection.recv(5000)
connection.close()
@ -462,8 +495,10 @@ class TestProxy(tservers.HTTPProxTest):
f = self.master.state.view[0]
assert f.server_conn.address == ("127.0.0.1", self.server.port)
class TestProxySSL(tservers.HTTPProxTest):
ssl=True
ssl = True
def test_request_ssl_setup_timestamp_presence(self):
# tests that the ssl timestamp is present when ssl is used
f = self.pathod("304:b@10k")
@ -479,16 +514,24 @@ class MasterRedirectRequest(tservers.TestMaster):
request = f.request
if request.path == "/p/201":
addr = f.live.c.server_conn.address
assert f.live.change_server(("127.0.0.1", self.redirect_port), ssl=False)
assert not f.live.change_server(("127.0.0.1", self.redirect_port), ssl=False)
tutils.raises("SSL handshake error", f.live.change_server, ("127.0.0.1", self.redirect_port), ssl=True)
assert f.live.change_server(
("127.0.0.1", self.redirect_port), ssl=False)
assert not f.live.change_server(
("127.0.0.1", self.redirect_port), ssl=False)
tutils.raises(
"SSL handshake error",
f.live.change_server,
("127.0.0.1",
self.redirect_port),
ssl=True)
assert f.live.change_server(addr, ssl=False)
request.url = "http://127.0.0.1:%s/p/201" % self.redirect_port
tservers.TestMaster.handle_request(self, f)
def handle_response(self, f):
f.response.content = str(f.client_conn.address.port)
f.response.headers["server-conn-id"] = [str(f.server_conn.source_address.port)]
f.response.headers[
"server-conn-id"] = [str(f.server_conn.source_address.port)]
tservers.TestMaster.handle_response(self, f)
@ -502,37 +545,41 @@ class TestRedirectRequest(tservers.HTTPProxTest):
self.server.clear_log()
self.server2.clear_log()
r1 = p.request("get:'%s/p/200'"%self.server.urlbase)
r1 = p.request("get:'%s/p/200'" % self.server.urlbase)
assert r1.status_code == 200
assert self.server.last_log()
assert not self.server2.last_log()
self.server.clear_log()
self.server2.clear_log()
r2 = p.request("get:'%s/p/201'"%self.server.urlbase)
r2 = p.request("get:'%s/p/201'" % self.server.urlbase)
assert r2.status_code == 201
assert not self.server.last_log()
assert self.server2.last_log()
self.server.clear_log()
self.server2.clear_log()
r3 = p.request("get:'%s/p/202'"%self.server.urlbase)
r3 = p.request("get:'%s/p/202'" % self.server.urlbase)
assert r3.status_code == 202
assert self.server.last_log()
assert not self.server2.last_log()
assert r1.content == r2.content == r3.content
assert r1.headers.get_first("server-conn-id") == r3.headers.get_first("server-conn-id")
assert r1.headers.get_first(
"server-conn-id") == r3.headers.get_first("server-conn-id")
# Make sure that we actually use the same connection in this test case
class MasterStreamRequest(tservers.TestMaster):
"""
Enables the stream flag on the flow for all requests
"""
def handle_responseheaders(self, f):
f.response.stream = True
f.reply()
class TestStreamRequest(tservers.HTTPProxTest):
masterclass = MasterStreamRequest
@ -541,7 +588,7 @@ class TestStreamRequest(tservers.HTTPProxTest):
# a request with 100k of data but without content-length
self.server.clear_log()
r1 = p.request("get:'%s/p/200:r:b@100k:d102400'"%self.server.urlbase)
r1 = p.request("get:'%s/p/200:r:b@100k:d102400'" % self.server.urlbase)
assert r1.status_code == 200
assert len(r1.content) > 100000
assert self.server.last_log()
@ -551,13 +598,13 @@ class TestStreamRequest(tservers.HTTPProxTest):
# simple request with streaming turned on
self.server.clear_log()
r1 = p.request("get:'%s/p/200'"%self.server.urlbase)
r1 = p.request("get:'%s/p/200'" % self.server.urlbase)
assert r1.status_code == 200
assert self.server.last_log()
# now send back 100k of data, streamed but not chunked
self.server.clear_log()
r1 = p.request("get:'%s/p/200:b@100k'"%self.server.urlbase)
r1 = p.request("get:'%s/p/200:b@100k'" % self.server.urlbase)
assert r1.status_code == 200
assert self.server.last_log()
@ -567,15 +614,27 @@ class TestStreamRequest(tservers.HTTPProxTest):
connection.connect(("127.0.0.1", self.proxy.port))
fconn = connection.makefile()
spec = '200:h"Transfer-Encoding"="chunked":r:b"4\\r\\nthis\\r\\n7\\r\\nisatest\\r\\n0\\r\\n\\r\\n"'
connection.send("GET %s/p/%s HTTP/1.1\r\n"%(self.server.urlbase, spec))
connection.send(
"GET %s/p/%s HTTP/1.1\r\n" %
(self.server.urlbase, spec))
connection.send("\r\n")
httpversion, code, msg, headers, content = http.read_response(fconn, "GET", None, include_body=False)
httpversion, code, msg, headers, content = http.read_response(
fconn, "GET", None, include_body=False)
assert headers["Transfer-Encoding"][0] == 'chunked'
assert code == 200
chunks = list(content for _, content, _ in http.read_http_body_chunked(fconn, headers, None, "GET", 200, False))
chunks = list(
content for _,
content,
_ in http.read_http_body_chunked(
fconn,
headers,
None,
"GET",
200,
False))
assert chunks == ["this", "isatest", ""]
connection.close()
@ -589,6 +648,7 @@ class MasterFakeResponse(tservers.TestMaster):
class TestFakeResponse(tservers.HTTPProxTest):
masterclass = MasterFakeResponse
def test_fake(self):
f = self.pathod("200")
assert "header_response" in f.headers.keys()
@ -601,6 +661,7 @@ class MasterKillRequest(tservers.TestMaster):
class TestKillRequest(tservers.HTTPProxTest):
masterclass = MasterKillRequest
def test_kill(self):
tutils.raises("server disconnect", self.pathod, "200")
# Nothing should have hit the server
@ -614,6 +675,7 @@ class MasterKillResponse(tservers.TestMaster):
class TestKillResponse(tservers.HTTPProxTest):
masterclass = MasterKillResponse
def test_kill(self):
tutils.raises("server disconnect", self.pathod, "200")
# The server should have seen a request
@ -627,6 +689,7 @@ class EResolver(tservers.TResolver):
class TestTransparentResolveError(tservers.TransparentProxTest):
resolver = EResolver
def test_resolve_error(self):
assert self.pathod("304").status_code == 502
@ -640,6 +703,7 @@ class MasterIncomplete(tservers.TestMaster):
class TestIncompleteResponse(tservers.HTTPProxTest):
masterclass = MasterIncomplete
def test_incomplete(self):
assert self.pathod("200").status_code == 502
@ -656,10 +720,16 @@ class TestUpstreamProxy(tservers.HTTPUpstreamProxTest, CommonMixin, AppMixin):
ssl = False
def test_order(self):
self.proxy.tmaster.replacehooks.add("~q", "foo", "bar") # replace in request
self.proxy.tmaster.replacehooks.add(
"~q",
"foo",
"bar") # replace in request
self.chain[0].tmaster.replacehooks.add("~q", "bar", "baz")
self.chain[1].tmaster.replacehooks.add("~q", "foo", "oh noes!")
self.chain[0].tmaster.replacehooks.add("~s", "baz", "ORLY") # replace in response
self.chain[0].tmaster.replacehooks.add(
"~s",
"baz",
"ORLY") # replace in response
p = self.pathoc()
req = p.request("get:'%s/p/418:b\"foo\"'" % self.server.urlbase)
@ -667,7 +737,10 @@ class TestUpstreamProxy(tservers.HTTPUpstreamProxTest, CommonMixin, AppMixin):
assert req.status_code == 418
class TestUpstreamProxySSL(tservers.HTTPUpstreamProxTest, CommonMixin, TcpMixin):
class TestUpstreamProxySSL(
tservers.HTTPUpstreamProxTest,
CommonMixin,
TcpMixin):
ssl = True
def _host_pattern_on(self, attr):
@ -677,7 +750,10 @@ class TestUpstreamProxySSL(tservers.HTTPUpstreamProxTest, CommonMixin, TcpMixin)
assert not hasattr(self, "_ignore_%s_backup" % attr)
backup = []
for proxy in self.chain:
old_matcher = getattr(proxy.tmaster.server.config, "check_%s" % attr)
old_matcher = getattr(
proxy.tmaster.server.config,
"check_%s" %
attr)
backup.append(old_matcher)
setattr(
proxy.tmaster.server.config,
@ -721,11 +797,14 @@ class TestUpstreamProxySSL(tservers.HTTPUpstreamProxTest, CommonMixin, TcpMixin)
assert req.content == "content"
assert req.status_code == 418
assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT from pathoc to chain[0],
# CONNECT from pathoc to chain[0],
assert self.proxy.tmaster.state.flow_count() == 2
# request from pathoc to chain[0]
assert self.chain[0].tmaster.state.flow_count() == 2 # CONNECT from proxy to chain[1],
# CONNECT from proxy to chain[1],
assert self.chain[0].tmaster.state.flow_count() == 2
# request from proxy to chain[1]
assert self.chain[1].tmaster.state.flow_count() == 1 # request from chain[0] (regular proxy doesn't store CONNECTs)
# request from chain[0] (regular proxy doesn't store CONNECTs)
assert self.chain[1].tmaster.state.flow_count() == 1
def test_closing_connect_response(self):
"""
@ -755,6 +834,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest):
def kill_requests(master, attr, exclude):
k = [0] # variable scope workaround: put into array
_func = getattr(master, attr)
def handler(f):
k[0] += 1
if not (k[0] in exclude):
@ -781,9 +861,11 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest):
p = self.pathoc()
req = p.request("get:'/p/418:b\"content\"'")
assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT and request
assert self.chain[0].tmaster.state.flow_count() == 4 # CONNECT, failing request,
# CONNECT, failing request,
assert self.chain[0].tmaster.state.flow_count() == 4
# reCONNECT, request
assert self.chain[1].tmaster.state.flow_count() == 2 # failing request, request
# failing request, request
assert self.chain[1].tmaster.state.flow_count() == 2
# (doesn't store (repeated) CONNECTs from chain[0]
# as it is a regular proxy)
assert req.content == "content"
@ -795,18 +877,26 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest):
assert self.proxy.tmaster.state.flows[0].request.form_in == "authority"
assert self.proxy.tmaster.state.flows[1].request.form_in == "relative"
assert self.chain[0].tmaster.state.flows[0].request.form_in == "authority"
assert self.chain[0].tmaster.state.flows[1].request.form_in == "relative"
assert self.chain[0].tmaster.state.flows[2].request.form_in == "authority"
assert self.chain[0].tmaster.state.flows[3].request.form_in == "relative"
assert self.chain[0].tmaster.state.flows[
0].request.form_in == "authority"
assert self.chain[0].tmaster.state.flows[
1].request.form_in == "relative"
assert self.chain[0].tmaster.state.flows[
2].request.form_in == "authority"
assert self.chain[0].tmaster.state.flows[
3].request.form_in == "relative"
assert self.chain[1].tmaster.state.flows[0].request.form_in == "relative"
assert self.chain[1].tmaster.state.flows[1].request.form_in == "relative"
assert self.chain[1].tmaster.state.flows[
0].request.form_in == "relative"
assert self.chain[1].tmaster.state.flows[
1].request.form_in == "relative"
req = p.request("get:'/p/418:b\"content2\"'")
assert req.status_code == 502
assert self.proxy.tmaster.state.flow_count() == 3 # + new request
assert self.chain[0].tmaster.state.flow_count() == 6 # + new request, repeated CONNECT from chain[1]
# + new request, repeated CONNECT from chain[1]
assert self.chain[0].tmaster.state.flow_count() == 6
# (both terminated)
assert self.chain[1].tmaster.state.flow_count() == 2 # nothing happened here
# nothing happened here
assert self.chain[1].tmaster.state.flow_count() == 2

Some files were not shown because too many files have changed in this diff Show More