Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-23 00:01:36 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of github.com:mitmproxy/mitmproxy

Browse Source
This commit is contained in:
Maximilian Hils 2015-05-31 01:51:31 +02:00
commit ec92fca8c0
122 changed files with 8494 additions and 1006 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.env
View File

@ -1,5 +1,5 @@
DIR=`dirname $0`
if [ -z "$VIRTUAL_ENV" ] && [ -f $DIR/../venv.mitmproxy/bin/activate ]; then
DIR="${0%/*}"
if [ -z "$VIRTUAL_ENV" ] && [ -f "$DIR/../venv.mitmproxy/bin/activate" ]; then
echo "Activating mitmproxy virtualenv..."
source $DIR/../venv.mitmproxy/bin/activate
source "$DIR/../venv.mitmproxy/bin/activate"
fi

20
check_coding_style.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
autopep8 -i -r -a -a .
if [[ -n "$(git status -s)" ]]; then
echo "autopep8 yielded the following changes:"
git status -s
git --no-pager diff
exit 1
fi
autoflake -i -r --remove-all-unused-imports --remove-unused-variables .
if [[ -n "$(git status -s)" ]]; then
echo "autoflake yielded the following changes:"
git status -s
git --no-pager diff
exit 1
fi
echo "Coding style seems to be ok."
exit 0

9
doc-src/01-bootstrap.min.css vendored
View File

File diff suppressed because one or more lines are too long

6706
doc-src/01-vendor.css Normal file
View File

File diff suppressed because it is too large Load Diff

43
doc-src/02-app.css Normal file
View File

@ -0,0 +1,43 @@
.masthead {
text-align: center;
border-bottom: 0;
}
.frontpage .talks div {
margin-bottom: 10px;
}
.nav-sidebar {
background-color: #f0f0f0;
margin-bottom: 20px;
}
.nav-sidebar li {
line-height: 1.1;
}
.nav-sidebar li > a,
.nav-sidebar .nav-header {
padding-left: 20px;
}
.nav-sidebar .nav-header {
margin-top: 1em;
font-size: 1.2em;
font-weight: bold;
}
.nav-sidebar .active > a,
.nav-sidebar .active > a:hover,
.nav-sidebar .active > a:focus {
color: #fff;
background-color: #428bca;
}
.tablenum {
font-weight: bold;
}
.nowrap {
white-space: nowrap;
}
.page-header {
margin: 0px 0 22px;
}
.page-header h1 {
margin-top: 0px;
}
/*# sourceMappingURL=02-app.css.map */

20
doc-src/02-docstyle.css
View File

@ -1,20 +0,0 @@
body {
padding-top: 60px;
padding-bottom: 40px;
}
.tablenum {
font-weight: bold;
}
.nowrap {
white-space: nowrap;
}
h1 {
line-height: 1.1;
}
.page-header {
margin: 0px 0 22px;
}

74
doc-src/_layout.html
View File

@ -1,36 +1,44 @@
<div class="navbar navbar-fixed-top">
<div class="navbar-inner">
<div class="container">
<a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</a>
<a class="brand" href="@!urlTo(idxpath)!@">mitmproxy $!VERSION!$ docs</a>
</div><!--/.nav-collapse -->
</div>
</div>
</div>
<div class="container">
<div class="row">
<div class="span3">
<div class="well sidebar-nav">
$!navbar!$
</div>
</div>
<div class="span9">
<div class="page-header">
<h1>@!this.title!@</h1>
<!DOCTYPE html>
<html>
<head>
<title>@!pageTitle!@</title>
$!header!$
</head>
<body>
<div class="navbar navbar-default navbar-static-top">
<div class="container">
<div class="navbar-header">
<a class="navbar-brand" href="@!urlTo("/index.html")!@">
<img height="20px" src="@!urlTo("mitmproxy-long.png")!@"/>
</a>
</div>
<div class="navbar-header navbar-right">
<a class="navbar-brand" hre="#">$!VERSION!$ docs</a>
</div>
</div>
</div>
$!body!$
</div>
</div>
<hr>
<footer>
<p>@!copyright!@</p>
</footer>
</div>
<div class="container">
<div class="row">
<div class="col-md-3">
$!navbar!$
</div>
<div class="col-md-9">
<div class="page-header">
<h1>@!this.title!@</h1>
</div>
$!body!$
</div>
</div>
</div>
<div class="container">
<hr>
<footer>
<p>@!copyright!@</p>
</footer>
</div>
</body>
</html>

2
doc-src/_nav.html
View File

@ -1,4 +1,4 @@
<ul class="nav nav-list">
<ul class="nav nav-sidebar">
$!nav(idxpath, this, state)!$
$!nav("install.html", this, state)!$
$!nav("certinstall.html", this, state)!$

42
doc-src/_websitelayout.html
View File

@ -1,42 +0,0 @@
<div class="navbar navbar-fixed-top">
<div class="navbar-inner">
<div class="container">
<a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</a>
<a class="brand" href="@!urlTo(idxpath)!@">mitmproxy</a>
<div class="nav">
<ul class="nav">
<li $!'class="active"' if this.match("/index.html", True) else ""!$> <a href="@!top!@/index.html">home</a> </li>
<li $!'class="active"' if this.under("/doc") else ""!$><a href="@!top!@/doc/index.html">docs</a></li>
<li $!'class="active"' if this.under("/about.html") else ""!$><a href="@!top!@/about.html">about</a></li>
</ul>
</div>
</div>
</div>
</div>
<div class="container">
<div class="row">
<div class="span3">
<div class="well sidebar-nav">
$!navbar!$
</div>
</div>
<div class="span9">
<div class="page-header">
<h1>@!this.title!@</h1>
</div>
$!body!$
</div>
</div>
<hr>
<footer>
<p>@!copyright!@</p>
</footer>
</div>

8
doc-src/dev/architecture.html
View File

@ -1,8 +1,8 @@
To give you a better understanding of how mitmproxy works, mitmproxy's high-level architecture is detailed
in the following graphic:
To give you a better understanding of how mitmproxy works, mitmproxy's
high-level architecture is detailed in the following graphic:
<img src="@!urlTo('schematics/architecture.png')!@">
<img class="img-responsive" src="@!urlTo('schematics/architecture.png')!@">
<a href="@!urlTo('schematics/architecture.pdf')!@">(architecture.pdf)</a>
<p>Please don't refrain from asking any further
questions on the mailing list, the IRC channel or the GitHub issue tracker.</p>
questions on the mailing list, the IRC channel or the GitHub issue tracker.</p>

8
doc-src/dev/index.py
View File

@ -1,8 +1,8 @@
from countershape import Page
pages = [
Page("testing.html", "Testing"),
Page("architecture.html", "Architecture"),
Page("sslkeylogfile.html", "TLS Master Secrets"),
# Page("addingviews.html", "Writing Content Views"),
Page("testing.html", "Testing"),
Page("architecture.html", "Architecture"),
Page("sslkeylogfile.html", "TLS Master Secrets"),
# Page("addingviews.html", "Writing Content Views"),
]

2
doc-src/features/index.py
View File

@ -16,4 +16,4 @@ pages = [
Page("tcpproxy.html", "TCP Proxy"),
Page("upstreamcerts.html", "Upstream Certs"),
Page("upstreamproxy.html", "Upstream proxy mode"),
]
]

8
doc-src/howmitmproxy.html
View File

@ -26,7 +26,7 @@ This is a proxy GET request - an extended form of the vanilla HTTP GET request
that includes a schema and host specification, and it includes all the
information mitmproxy needs to proceed.
<img src="explicit.png"/>
<img class="img-responsive" src="explicit.png"/>
<table class="table">
<tbody>
@ -158,7 +158,7 @@ handshake. Luckily, this is almost never an issue in practice.
Lets put all of this together into the complete explicitly proxied HTTPS flow.
<img src="explicit_https.png"/>
<img class="img-responsive" src="explicit_https.png"/>
<table class="table">
<tbody>
@ -250,7 +250,7 @@ mitmproxy, this takes the form of a built-in set of
that know how to talk to each platform's redirection mechanism. Once we have
this information, the process is fairly straight-forward.
<img src="transparent.png"/>
<img class="img-responsive" src="transparent.png"/>
<table class="table">
@ -296,7 +296,7 @@ transparently proxying HTTP, and explicitly proxying HTTPS. We use the routing
mechanism to establish the upstream server address, and then proceed as for
explicit HTTPS connections to establish the CN and SANs, and cope with SNI.
<img src="transparent_https.png"/>
<img class="img-responsive" src="transparent_https.png"/>
<table class="table">

11
doc-src/index.py
View File

@ -2,7 +2,7 @@ import os
import sys
import datetime
import countershape
from countershape import Page, Directory, markup, model
from countershape import Page, Directory, markup
import countershape.template
MITMPROXY_SRC = os.path.abspath(
@ -15,10 +15,10 @@ ns.VERSION = version.VERSION
if ns.options.website:
ns.idxpath = "doc/index.html"
this.layout = countershape.Layout("_websitelayout.html")
else:
ns.idxpath = "index.html"
this.layout = countershape.Layout("_layout.html")
this.layout = countershape.layout.FileLayout("_layout.html")
ns.title = countershape.template.Template(None, "<h1>@!this.title!@</h1>")
this.titlePrefix = "%s - " % version.NAMEVERSION
@ -37,7 +37,7 @@ def mpath(p):
def example(s):
d = file(mpath(s)).read().rstrip()
extemp = """<div class="example">%s<div class="example_legend">(%s)</div></div>"""
return extemp%(countershape.template.Syntax("py")(d), s)
return extemp % (countershape.template.Syntax("py")(d), s)
ns.example = example
@ -52,7 +52,8 @@ def nav(page, current, state):
else:
pre = "<li>"
p = state.application.getPage(page)
return pre + '<a href="%s">%s</a></li>'%(model.UrlTo(page), p.title)
return pre + \
'<a href="%s">%s</a></li>' % (countershape.widgets.UrlTo(page), p.title)
ns.nav = nav
ns.navbar = countershape.template.File(None, "_nav.html")

BIN
doc-src/mitmproxy-long.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 121 KiB

28
doc-src/mitmproxy.html
View File

@ -7,13 +7,13 @@ documentation from any __mitmproxy__ screen.
## Flow list
The flow list shows an index of captured flows in chronological order.
The flow list shows an index of captured flows in chronological order.
<img src="@!urlTo('screenshots/mitmproxy.png')!@"/>
<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy.png')!@"/>
- __1__: A GET request, returning a 302 Redirect response.
- __2__: A GET request, returning 16.75kb of text/html data.
- __3__: A replayed request.
- __3__: A replayed request.
- __4__: Intercepted flows are indicated with orange text. The user may edit
these flows, and then accept them (using the _a_ key) to continue. In this
case, the request has been intercepted on the way to the server.
@ -32,7 +32,7 @@ interfaces.
The __Flow View__ lets you inspect and manipulate a single flow:
<img src="@!urlTo('screenshots/mitmproxy-flowview.png')!@"/>
<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-flowview.png')!@"/>
- __1__: Flow summary.
- __2__: The Request/Response tabs, showing you which part of the flow you are
@ -53,11 +53,11 @@ using the _m_ key.
Much of the data that we'd like to interact with in mitmproxy is structured.
For instance, headers, queries and form data can all be thought of as a list of
key/value pairs. Mitmproxy has a built-in editor that lays this type of data
out in a grid for easy manipulation.
out in a grid for easy manipulation.
At the moment, the Grid Editor is used in four parts of mitmproxy:
- Editing request or response headers (_e_ for edit, then _h_ for headers in flow view)
- Editing request or response headers (_e_ for edit, then _h_ for headers in flow view)
- Editing a query string (_e_ for edit, then _q_ for query in flow view)
- Editing a URL-encoded form (_e_ for edit, then _f_ for form in flow view)
- Editing replacement patterns (_R_ globally)
@ -65,13 +65,13 @@ At the moment, the Grid Editor is used in four parts of mitmproxy:
If there is is no data, an empty editor will be started to let you add some.
Here is the editor showing the headers from a request:
<img src="@!urlTo('screenshots/mitmproxy-kveditor.png')!@"/>
<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-kveditor.png')!@"/>
To edit, navigate to the key or value you want to modify using the arrow or vi
navigation keys, and press enter. The background color will change to show that
you are in edit mode for the specified field:
<img src="@!urlTo('screenshots/mitmproxy-kveditor-editmode.png')!@"/>
<img class="img-responsive" src="@!urlTo('screenshots/mitmproxy-kveditor-editmode.png')!@"/>
Modify the field as desired, then press escape to exit edit mode when you're
done. You can also add a row (_a_ key), delete a row (_d_ key), spawn an
@ -83,12 +83,12 @@ help (_?_ key) for more.
__mitmproxy__'s interception functionality lets you pause an HTTP request or
response, inspect and modify it, and then accept it to send it on to the server
or client.
or client.
### 1: Set an interception pattern
<img src="@!urlTo('mitmproxy-intercept-filt.png')!@"/>
<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-filt.png')!@"/>
We press _i_ to set an interception pattern. In this case, the __~q__ filter
pattern tells __mitmproxy__ to intercept all requests. For complete filter
@ -97,19 +97,19 @@ document, or the built-in help function in __mitmproxy__.
### 2: Intercepted connections are indicated with orange text:
<img src="@!urlTo('mitmproxy-intercept-mid.png')!@"/>
<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-mid.png')!@"/>
### 3: You can now view and modify the request:
<img src="@!urlTo('mitmproxy-intercept-options.png')!@"/>
<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-options.png')!@"/>
In this case, we viewed the request by selecting it, pressed _e_ for "edit"
and _m_ for "method" to change the HTTP request method.
### 4: Accept the intercept to continue:
<img src="@!urlTo('mitmproxy-intercept-result.png')!@"/>
<img class="img-responsive" src="@!urlTo('mitmproxy-intercept-result.png')!@"/>
Finally, we press _a_ to accept the modified request, which is then sent on to
the server. In this case, we changed the request from an HTTP GET to
OPTIONS, and Google's server has responded with a 405 "Method not allowed".
OPTIONS, and Google's server has responded with a 405 "Method not allowed".

20
doc-src/modes.html
View File

@ -9,7 +9,7 @@ variety of scenarios:
Now, which one should you pick? Use this flow chart:
<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/>
<div class="page-header">
<h1>Regular Proxy</h1>
@ -31,7 +31,7 @@ these cases, you need to use mitmproxy's transparent mode.
If you are proxying an external device, your network will probably look like this:
<img src="@!urlTo('schematics/proxy-modes-regular.png')!@">
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-regular.png')!@">
The square brackets signify the source and destination IP addresses. Your
client explicitly connects to mitmproxy and mitmproxy explicitly connects
@ -48,7 +48,7 @@ below, a machine running mitmproxy has been inserted between the router and
the internet:
<a href="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
<img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@">
</a>
The square brackets signify the source and destination IP addresses. Round
@ -60,7 +60,7 @@ remove the target information, leaving mitmproxy unable to determine the real
destination.
<a href="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@">
<img src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a>
<h2>Common Configurations</h2>
@ -79,7 +79,7 @@ intact, is to simply configure the client with the mitmproxy box as the
default gateway.
<a href="@!urlTo('schematics/proxy-modes-transparent-2.png')!@">
<img src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a>
In this scenario, we would:
@ -98,7 +98,7 @@ for your device.
Setting the custom gateway on clients can be automated by serving the settings
out to clients over DHCP. This lets set up an interception network where all
clients are proxied automatically, which can save time and effort.
clients are proxied automatically, which can save time and effort.
<div class="well">
@ -141,7 +141,7 @@ packet filter you're using. In most cases, the configuration will look like
this:
<a href="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
<img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@">
</a>
@ -154,7 +154,7 @@ Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal
HTTP server:
<a href="@!urlTo('schematics/proxy-modes-reverse.png')!@">
<img src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-reverse.png')!@">
</a>
There are various use-cases:
@ -215,8 +215,8 @@ appliance, you can use mitmproxy's upstream mode. In upstream mode, all
requests are unconditionally transferred to an upstream proxy of your choice.
<a href="@!urlTo('schematics/proxy-modes-upstream.png')!@">
<img src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
<img class="img-responsive" src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a>
mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy
mode. You could in theory chain multiple mitmproxy instances in a row, but
that doesn't make any sense in practice (i.e. outside of our tests).
that doesn't make any sense in practice (i.e. outside of our tests).

6
doc-src/tutorials/gamecenter.html
View File

@ -15,14 +15,14 @@ Worm](http://itunes.apple.com/us/app/super-mega-worm/id388541990?mt=8) - a
great little retro-apocalyptic sidescroller for the iPhone:
<center>
<img src="@!urlTo('tutorials/supermega.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/supermega.png')!@"/>
</center>
After finishing a game (take your time), watch the traffic flowing through
mitmproxy:
<center>
<img src="@!urlTo('tutorials/one.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/one.png')!@"/>
</center>
We see a bunch of things we might expect - initialisation, the retrieval of
@ -97,7 +97,7 @@ replay.
## The glorious result and some intrigue
<center>
<img src="@!urlTo('tutorials/leaderboard.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/leaderboard.png')!@"/>
</center>
And that's it - according to the records, I am the greatest Super Mega Worm

2
doc-src/tutorials/index.py
View File

@ -4,4 +4,4 @@ pages = [
Page("30second.html", "Client playback: a 30 second example"),
Page("gamecenter.html", "Setting highscores on Apple's GameCenter"),
Page("transparent-dhcp.html", "Transparently proxify virtual machines")
]
]

8
doc-src/tutorials/transparent-dhcp.html
View File

@ -4,10 +4,10 @@ This walkthrough illustrates how to set up transparent proxying with mitmproxy.
The network setup is simple: `internet <--> proxy vm <--> (virtual) internal network`.
For the proxy machine, *eth0* represents the outgoing network. *eth1* is connected to the internal network that will be proxified, using a static ip (192.168.3.1).
<hr>VirtualBox configuration:
<img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth0.png')!@"/><br><br>
<img src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth1.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth0.png')!@"/><br><br>
<img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_vbox_eth1.png')!@"/>
<br>Proxy VM:
<img src="@!urlTo('tutorials/transparent-dhcp/step1_proxy.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step1_proxy.png')!@"/>
<hr>
2. **Configure DHCP and DNS**
We use dnsmasq to provide DHCP and DNS in our internal network.
@ -34,7 +34,7 @@ This walkthrough illustrates how to set up transparent proxying with mitmproxy.
`sudo service dnsmasq restart`
<hr>
Your proxied machine's network settings should now look similar to this:
<img src="@!urlTo('tutorials/transparent-dhcp/step2_proxied_vm.png')!@"/>
<img class="img-responsive" src="@!urlTo('tutorials/transparent-dhcp/step2_proxied_vm.png')!@"/>
<hr>
3. **Set up traffic redirection to mitmproxy**

2
examples/add_header.py
View File

@ -1,2 +1,2 @@
def response(context, flow):
flow.response.headers["newheader"] = ["foo"]
flow.response.headers["newheader"] = ["foo"]

14
examples/change_upstream_proxy.py
View File

@ -1,10 +1,13 @@
# This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy
# in upstream proxy mode.
#
# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s "change_upstream_proxy.py host"
# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s
# "change_upstream_proxy.py host"
from libmproxy.protocol.http import send_connect_request
alternative_upstream_proxy = ("localhost", 8082)
def should_redirect(flow):
return flow.request.host == "example.com"
@ -15,7 +18,12 @@ def request(context, flow):
# If you want to change the target server, you should modify flow.request.host and flow.request.port
# flow.live.change_server should only be used by inline scripts to change the upstream proxy,
# unless you are sure that you know what you are doing.
server_changed = flow.live.change_server(alternative_upstream_proxy, persistent_change=True)
server_changed = flow.live.change_server(
alternative_upstream_proxy,
persistent_change=True)
if flow.request.scheme == "https" and server_changed:
send_connect_request(flow.live.c.server_conn, flow.request.host, flow.request.port)
send_connect_request(
flow.live.c.server_conn,
flow.request.host,
flow.request.port)
flow.live.c.establish_ssl(server=True)

8
examples/dns_spoofing.py
View File

@ -25,11 +25,13 @@ mitmproxy -p 443 -R https2http://localhost:8000
def request(context, flow):
if flow.client_conn.ssl_established:
# TLS SNI or Host header
flow.request.host = flow.client_conn.connection.get_servername() or flow.request.pretty_host(hostheader=True)
flow.request.host = flow.client_conn.connection.get_servername(
) or flow.request.pretty_host(hostheader=True)
# If you use a https2http location as default destination, these attributes need to be corrected as well:
# If you use a https2http location as default destination, these
# attributes need to be corrected as well:
flow.request.port = 443
flow.request.scheme = "https"
else:
# Host header
flow.request.host = flow.request.pretty_host(hostheader=True)
flow.request.host = flow.request.pretty_host(hostheader=True)

2
examples/dup_and_replay.py
View File

@ -1,4 +1,4 @@
def request(context, flow):
f = context.duplicate_flow(flow)
f.request.path = "/changed"
context.replay_request(f)
context.replay_request(f)

14
examples/filt.py
View File

@ -3,12 +3,14 @@
from libmproxy import filt
def start(context, argv):
if len(argv) != 2:
raise ValueError("Usage: -s 'filt.py FILTER'")
context.filter = filt.parse(argv[1])
if len(argv) != 2:
raise ValueError("Usage: -s 'filt.py FILTER'")
context.filter = filt.parse(argv[1])
def response(context, flow):
if flow.match(context.filter):
print("Flow matches filter:")
print(flow)
if flow.match(context.filter):
print("Flow matches filter:")
print(flow)

3
examples/flowbasic
View File

@ -36,7 +36,8 @@ class MyMaster(flow.FlowMaster):
config = proxy.ProxyConfig(
port=8080,
cadir="~/.mitmproxy/" # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
# use ~/.mitmproxy/mitmproxy-ca.pem as default CA file.
cadir="~/.mitmproxy/"
)
state = flow.State()
server = ProxyServer(config)

2
examples/flowwriter.py
View File

@ -17,4 +17,4 @@ def start(context, argv):
def response(context, flow):
if random.choice([True, False]):
context.flow_writer.add(flow)
context.flow_writer.add(flow)

88
examples/har_extractor.py
View File

@ -83,7 +83,8 @@ def response(context, flow):
# Calculate the connect_time for this server_conn. Afterwards add it to
# seen list, in order to avoid the connect_time being present in entries
# that use an existing connection.
connect_time = flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start
connect_time = flow.server_conn.timestamp_tcp_setup - \
flow.server_conn.timestamp_start
context.seen_server.add(flow.server_conn)
if flow.server_conn.timestamp_ssl_setup is not None:
@ -91,7 +92,8 @@ def response(context, flow):
# the start of the successful tcp setup and the successful ssl
# setup. If no ssl setup has been made it is left as -1 since it
# doesn't apply to this connection.
ssl_time = flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup
ssl_time = flow.server_conn.timestamp_ssl_setup - \
flow.server_conn.timestamp_tcp_setup
# Calculate the raw timings from the different timestamps present in the
# request and response object. For lack of a way to measure it dns timings
@ -110,7 +112,8 @@ def response(context, flow):
# HAR timings are integers in ms, so we have to re-encode the raw timings to
# that format.
timings = dict([(key, int(1000 * value)) for key, value in timings_raw.iteritems()])
timings = dict([(key, int(1000 * value))
for key, value in timings_raw.iteritems()])
# The full_time is the sum of all timings. Timings set to -1 will be ignored
# as per spec.
@ -119,20 +122,27 @@ def response(context, flow):
if item > -1:
full_time += item
started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, tz=utc).isoformat()
started_date_time = datetime.fromtimestamp(
flow.request.timestamp_start,
tz=utc).isoformat()
request_query_string = [{"name": k, "value": v} for k, v in flow.request.get_query()]
request_query_string = [{"name": k, "value": v}
for k, v in flow.request.get_query()]
request_http_version = ".".join([str(v) for v in flow.request.httpversion])
# Cookies are shaped as tuples by MITMProxy.
request_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.request.get_cookies() or {}).iteritems()]
request_cookies = [{"name": k.strip(), "value": v[0]}
for k, v in (flow.request.get_cookies() or {}).iteritems()]
request_headers = [{"name": k, "value": v} for k, v in flow.request.headers]
request_headers_size = len(str(flow.request.headers))
request_body_size = len(flow.request.content)
response_http_version = ".".join([str(v) for v in flow.response.httpversion])
response_http_version = ".".join(
[str(v) for v in flow.response.httpversion])
# Cookies are shaped as tuples by MITMProxy.
response_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.response.get_cookies() or {}).iteritems()]
response_headers = [{"name": k, "value": v} for k, v in flow.response.headers]
response_cookies = [{"name": k.strip(), "value": v[0]}
for k, v in (flow.response.get_cookies() or {}).iteritems()]
response_headers = [{"name": k, "value": v}
for k, v in flow.response.headers]
response_headers_size = len(str(flow.response.headers))
response_body_size = len(flow.response.content)
response_body_decoded_size = len(flow.response.get_decoded_content())
@ -140,33 +150,43 @@ def response(context, flow):
response_mime_type = flow.response.headers.get_first('Content-Type', '')
response_redirect_url = flow.response.headers.get_first('Location', '')
entry = HAR.entries({"startedDateTime": started_date_time,
"time": full_time,
"request": {"method": flow.request.method,
"url": flow.request.url,
"httpVersion": request_http_version,
"cookies": request_cookies,
"headers": request_headers,
"queryString": request_query_string,
"headersSize": request_headers_size,
"bodySize": request_body_size, },
"response": {"status": flow.response.code,
"statusText": flow.response.msg,
"httpVersion": response_http_version,
"cookies": response_cookies,
"headers": response_headers,
"content": {"size": response_body_size,
"compression": response_body_compression,
"mimeType": response_mime_type},
"redirectURL": response_redirect_url,
"headersSize": response_headers_size,
"bodySize": response_body_size, },
"cache": {},
"timings": timings, })
entry = HAR.entries(
{
"startedDateTime": started_date_time,
"time": full_time,
"request": {
"method": flow.request.method,
"url": flow.request.url,
"httpVersion": request_http_version,
"cookies": request_cookies,
"headers": request_headers,
"queryString": request_query_string,
"headersSize": request_headers_size,
"bodySize": request_body_size,
},
"response": {
"status": flow.response.code,
"statusText": flow.response.msg,
"httpVersion": response_http_version,
"cookies": response_cookies,
"headers": response_headers,
"content": {
"size": response_body_size,
"compression": response_body_compression,
"mimeType": response_mime_type},
"redirectURL": response_redirect_url,
"headersSize": response_headers_size,
"bodySize": response_body_size,
},
"cache": {},
"timings": timings,
})
# If the current url is in the page list of context.HARLog or does not have
# a referrer we add it as a new pages object.
if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get('Referer', None) is None:
if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get(
'Referer',
None) is None:
page_id = context.HARLog.create_page_id()
context.HARLog.add(
HAR.pages({
@ -231,4 +251,4 @@ def print_attributes(obj, filter_string=None, hide_privates=False):
if filter_string is not None and filter_string not in attr:
continue
value = getattr(obj, attr)
print "%s.%s" % ('obj', attr), value, type(value)
print "%s.%s" % ('obj', attr), value, type(value)

9
examples/iframe_injector.py
View File

@ -16,7 +16,12 @@ def response(context, flow):
with decoded(flow.response): # Remove content encoding (gzip, ...)
html = BeautifulSoup(flow.response.content)
if html.body:
iframe = html.new_tag("iframe", src=context.iframe_url, frameborder=0, height=0, width=0)
iframe = html.new_tag(
"iframe",
src=context.iframe_url,
frameborder=0,
height=0,
width=0)
html.body.insert(0, iframe)
flow.response.content = str(html)
context.log("Iframe inserted.")
context.log("Iframe inserted.")

3
examples/ignore_websocket.py
View File

@ -24,6 +24,7 @@ def done(context):
HTTPRequest._headers_to_strip_off.append("Connection")
HTTPRequest._headers_to_strip_off.append("Upgrade")
@concurrent
def response(context, flow):
value = flow.response.headers.get_first("Connection", None)
@ -32,4 +33,4 @@ def response(context, flow):
flow.client_conn.send(flow.response.assemble())
# ...and then delegate to tcp passthrough.
TCPHandler(flow.live.c, log=False).handle_messages()
flow.reply(KILL)
flow.reply(KILL)

60
examples/mitmproxywrapper.py
View File

@ -14,23 +14,29 @@ import contextlib
import os
import sys
class Wrapper(object):
def __init__(self, port, extra_arguments=None):
self.port = port
self.extra_arguments = extra_arguments
def run_networksetup_command(self, *arguments):
return subprocess.check_output(['sudo', 'networksetup'] + list(arguments))
return subprocess.check_output(
['sudo', 'networksetup'] + list(arguments))
def proxy_state_for_service(self, service):
state = self.run_networksetup_command('-getwebproxy', service).splitlines()
state = self.run_networksetup_command(
'-getwebproxy',
service).splitlines()
return dict([re.findall(r'([^:]+): (.*)', line)[0] for line in state])
def enable_proxy_for_service(self, service):
print 'Enabling proxy on {}...'.format(service)
for subcommand in ['-setwebproxy', '-setsecurewebproxy']:
self.run_networksetup_command(subcommand, service, '127.0.0.1', str(self.port))
self.run_networksetup_command(
subcommand, service, '127.0.0.1', str(
self.port))
def disable_proxy_for_service(self, service):
print 'Disabling proxy on {}...'.format(service)
@ -39,14 +45,20 @@ class Wrapper(object):
def interface_name_to_service_name_map(self):
order = self.run_networksetup_command('-listnetworkserviceorder')
mapping = re.findall(r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$', order, re.MULTILINE)
mapping = re.findall(
r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$',
order,
re.MULTILINE)
return dict([(b, a) for (a, b) in mapping])
def run_command_with_input(self, command, input):
popen = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
popen = subprocess.Popen(
command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
(stdout, stderr) = popen.communicate(input)
return stdout
def primary_interace_name(self):
scutil_script = 'get State:/Network/Global/IPv4\nd.show\n'
stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
@ -54,13 +66,15 @@ class Wrapper(object):
return interface
def primary_service_name(self):
return self.interface_name_to_service_name_map()[self.primary_interace_name()]
return self.interface_name_to_service_name_map()[
self.primary_interace_name()]
def proxy_enabled_for_service(self, service):
return self.proxy_state_for_service(service)['Enabled'] == 'Yes'
def toggle_proxy(self):
new_state = not self.proxy_enabled_for_service(self.primary_service_name())
new_state = not self.proxy_enabled_for_service(
self.primary_service_name())
for service_name in self.connected_service_names():
if self.proxy_enabled_for_service(service_name) and not new_state:
self.disable_proxy_for_service(service_name)
@ -74,8 +88,11 @@ class Wrapper(object):
service_names = []
for service_id in service_ids:
scutil_script = 'show Setup:/Network/Service/{}\n'.format(service_id)
stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script)
scutil_script = 'show Setup:/Network/Service/{}\n'.format(
service_id)
stdout = self.run_command_with_input(
'/usr/sbin/scutil',
scutil_script)
service_name, = re.findall(r'UserDefinedName\s*:\s*(.+)', stdout)
service_names.append(service_name)
@ -102,7 +119,7 @@ class Wrapper(object):
for service_name in connected_service_names:
if not self.proxy_enabled_for_service(service_name):
self.enable_proxy_for_service(service_name)
yield
for service_name in connected_service_names:
@ -119,15 +136,23 @@ class Wrapper(object):
def main(cls):
parser = argparse.ArgumentParser(
description='Helper tool for OS X proxy configuration and mitmproxy.',
epilog='Any additional arguments will be passed on unchanged to mitmproxy.'
)
parser.add_argument('-t', '--toggle', action='store_true', help='just toggle the proxy configuration')
epilog='Any additional arguments will be passed on unchanged to mitmproxy.')
parser.add_argument(
'-t',
'--toggle',
action='store_true',
help='just toggle the proxy configuration')
# parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy')
parser.add_argument('-p', '--port', type=int, help='override the default port of 8080', default=8080)
parser.add_argument(
'-p',
'--port',
type=int,
help='override the default port of 8080',
default=8080)
args, extra_arguments = parser.parse_known_args()
wrapper = cls(port=args.port, extra_arguments=extra_arguments)
if args.toggle:
wrapper.toggle_proxy()
# elif args.honeyproxy:
@ -139,4 +164,3 @@ class Wrapper(object):
if __name__ == '__main__':
Wrapper.ensure_superuser()
Wrapper.main()

5
examples/modify_form.py
View File

@ -1,6 +1,7 @@
def request(context, flow):
if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]:
if "application/x-www-form-urlencoded" in flow.request.headers[
"content-type"]:
form = flow.request.get_form_urlencoded()
form["mitmproxy"] = ["rocks"]
flow.request.set_form_urlencoded(form)
flow.request.set_form_urlencoded(form)

2
examples/modify_querystring.py
View File

@ -3,4 +3,4 @@ def request(context, flow):
q = flow.request.get_query()
if q:
q["mitmproxy"] = ["rocks"]
flow.request.set_query(q)
flow.request.set_query(q)

7
examples/modify_response_body.py
View File

@ -6,10 +6,13 @@ from libmproxy.protocol.http import decoded
def start(context, argv):
if len(argv) != 3:
raise ValueError('Usage: -s "modify-response-body.py old new"')
# You may want to use Python's argparse for more sophisticated argument parsing.
# You may want to use Python's argparse for more sophisticated argument
# parsing.
context.old, context.new = argv[1], argv[2]
def response(context, flow):
with decoded(flow.response): # automatically decode gzipped responses.
flow.response.content = flow.response.content.replace(context.old, context.new)
flow.response.content = flow.response.content.replace(
context.old,
context.new)

2
examples/nonblocking.py
View File

@ -6,4 +6,4 @@ from libmproxy.script import concurrent
def request(context, flow):
print "handle request: %s%s" % (flow.request.host, flow.request.path)
time.sleep(5)
print "start request: %s%s" % (flow.request.host, flow.request.path)
print "start request: %s%s" % (flow.request.host, flow.request.path)

2
examples/proxapp.py
View File

@ -21,4 +21,4 @@ def start(context, argv):
# SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design.
# mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set)
# but won't send any data.
context.app_registry.add(app, "example.com", 443)
context.app_registry.add(app, "example.com", 443)

5
examples/read_dumpfile
View File

@ -4,7 +4,8 @@
#
from libmproxy import flow
import json, sys
import json
import sys
with open("logfile", "rb") as logfile:
freader = flow.FlowReader(logfile)
@ -14,5 +15,5 @@ with open("logfile", "rb") as logfile:
print(f.request.host)
json.dump(f.get_state(), sys.stdout, indent=4)
print ""
except flow.FlowReadError, v:
except flow.FlowReadError as v:
print "Flow file corrupted. Stopped loading."

3
examples/redirect_requests.py
View File

@ -8,7 +8,8 @@ This example shows two ways to redirect flows to other destinations.
def request(context, flow):
# pretty_host(hostheader=True) takes the Host: header of the request into account,
# which is useful in transparent mode where we usually only have the IP otherwise.
# which is useful in transparent mode where we usually only have the IP
# otherwise.
# Method 1: Answer with a locally generated response
if flow.request.pretty_host(hostheader=True).endswith("example.com"):

2
examples/stream.py
View File

@ -2,4 +2,4 @@ def responseheaders(context, flow):
"""
Enables streaming for all responses.
"""
flow.response.stream = True
flow.response.stream = True

4
examples/stream_modify.py
View File

@ -11,7 +11,7 @@ Be aware that content replacement isn't trivial:
def modify(chunks):
"""
chunks is a generator that can be used to iterate over all chunks.
Each chunk is a (prefix, content, suffix) tuple.
Each chunk is a (prefix, content, suffix) tuple.
For example, in the case of chunked transfer encoding: ("3\r\n","foo","\r\n")
"""
for prefix, content, suffix in chunks:
@ -19,4 +19,4 @@ def modify(chunks):
def responseheaders(context, flow):
flow.response.stream = modify
flow.response.stream = modify

9
examples/stub.py
View File

@ -1,12 +1,15 @@
"""
This is a script stub, with definitions for all events.
"""
def start(context, argv):
"""
Called once on script startup, before any other events.
"""
context.log("start")
def clientconnect(context, conn_handler):
"""
Called when a client initiates a connection to the proxy. Note that a
@ -14,6 +17,7 @@ def clientconnect(context, conn_handler):
"""
context.log("clientconnect")
def serverconnect(context, conn_handler):
"""
Called when the proxy initiates a connection to the target server. Note that a
@ -21,6 +25,7 @@ def serverconnect(context, conn_handler):
"""
context.log("serverconnect")
def request(context, flow):
"""
Called when a client request has been received.
@ -36,12 +41,14 @@ def responseheaders(context, flow):
"""
context.log("responseheaders")
def response(context, flow):
"""
Called when a server response has been received.
"""
context.log("response")
def error(context, flow):
"""
Called when a flow error has occured, e.g. invalid server responses, or
@ -50,12 +57,14 @@ def error(context, flow):
"""
context.log("error")
def clientdisconnect(context, conn_handler):
"""
Called when a client disconnects from the proxy.
"""
context.log("clientdisconnect")
def done(context):
"""
Called once on script shutdown, after any other events.

3
examples/upsidedownternet.py
View File

@ -2,6 +2,7 @@ import cStringIO
from PIL import Image
from libmproxy.protocol.http import decoded
def response(context, flow):
if flow.response.headers.get_first("content-type", "").startswith("image"):
with decoded(flow.response): # automatically decode gzipped responses.
@ -13,4 +14,4 @@ def response(context, flow):
flow.response.content = s2.getvalue()
flow.response.headers["content-type"] = ["image/png"]
except: # Unknown image types etc.
pass
pass

20
libmproxy/cmdline.py
View File

@ -65,7 +65,7 @@ def parse_replace_hook(s):
patt, regex, replacement = _parse_hook(s)
try:
re.compile(regex)
except re.error, e:
except re.error as e:
raise ParseException("Malformed replacement regex: %s" % str(e.message))
return patt, regex, replacement
@ -127,7 +127,6 @@ def parse_server_spec_special(url):
return ret
def get_common_options(options):
stickycookie, stickyauth = None, None
if options.stickycookie_filt:
@ -142,17 +141,17 @@ def get_common_options(options):
for i in options.replace:
try:
p = parse_replace_hook(i)
except ParseException, e:
except ParseException as e:
raise configargparse.ArgumentTypeError(e.message)
reps.append(p)
for i in options.replace_file:
try:
patt, rex, path = parse_replace_hook(i)
except ParseException, e:
except ParseException as e:
raise configargparse.ArgumentTypeError(e.message)
try:
v = open(path, "rb").read()
except IOError, e:
except IOError as e:
raise configargparse.ArgumentTypeError(
"Could not read replace file: %s" % path
)
@ -162,7 +161,7 @@ def get_common_options(options):
for i in options.setheader:
try:
p = parse_setheader(i)
except ParseException, e:
except ParseException as e:
raise configargparse.ArgumentTypeError(e.message)
setheaders.append(p)
@ -221,7 +220,7 @@ def common_options(parser):
parser.add_argument(
"--cadir",
action="store", type=str, dest="cadir", default=config.CA_DIR,
help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR
help="Location of the default mitmproxy CA files. (%s)" % config.CA_DIR
)
parser.add_argument(
"--host",
@ -482,9 +481,10 @@ def common_options(parser):
)
group.add_argument(
"--replay-ignore-host",
action="store_true", dest="replay_ignore_host", default=False,
help="Ignore request's destination host while searching for a saved flow to replay"
)
action="store_true",
dest="replay_ignore_host",
default=False,
help="Ignore request's destination host while searching for a saved flow to replay")
group = parser.add_argument_group(
"Replacements",

29
libmproxy/console/__init__.py
View File

@ -85,10 +85,10 @@ class ConsoleState(flow.State):
return self.view[pos], pos
def get_next(self, pos):
return self.get_from_pos(pos+1)
return self.get_from_pos(pos + 1)
def get_prev(self, pos):
return self.get_from_pos(pos-1)
return self.get_from_pos(pos - 1)
def delete_flow(self, f):
if f in self.view and self.view.index(f) <= self.focus:
@ -255,7 +255,7 @@ class ConsoleMaster(flow.FlowMaster):
try:
f = file(path, mode)
self.start_stream(f, None)
except IOError, v:
except IOError as v:
return str(v)
self.stream_path = path
@ -263,22 +263,24 @@ class ConsoleMaster(flow.FlowMaster):
status, val = s.run(method, f)
if val:
if status:
self.add_event("Method %s return: %s"%(method, val), "debug")
self.add_event("Method %s return: %s" % (method, val), "debug")
else:
self.add_event("Method %s error: %s"%(method, val[1]), "error")
self.add_event(
"Method %s error: %s" %
(method, val[1]), "error")
def run_script_once(self, command, f):
if not command:
return
self.add_event("Running script on flow: %s"%command, "debug")
self.add_event("Running script on flow: %s" % command, "debug")
try:
s = script.Script(command, self)
except script.ScriptError, v:
except script.ScriptError as v:
signals.status_message.send(
message = "Error loading script."
)
self.add_event("Error loading script:\n%s"%v.args[0], "error")
self.add_event("Error loading script:\n%s" % v.args[0], "error")
return
if f.request:
@ -562,7 +564,7 @@ class ConsoleMaster(flow.FlowMaster):
for i in flows:
fw.add(i)
f.close()
except IOError, v:
except IOError as v:
signals.status_message.send(message=v.strerror)
def save_one_flow(self, path, flow):
@ -575,13 +577,13 @@ class ConsoleMaster(flow.FlowMaster):
if not path:
return
ret = self.load_flows_path(path)
return ret or "Flows loaded from %s"%path
return ret or "Flows loaded from %s" % path
def load_flows_path(self, path):
reterr = None
try:
flow.FlowMaster.load_flows_file(self, path)
except flow.FlowReadError, v:
except flow.FlowReadError as v:
reterr = str(v)
signals.flowlist_change.send(self)
return reterr
@ -652,7 +654,8 @@ class ConsoleMaster(flow.FlowMaster):
)
def process_flow(self, f):
if self.state.intercept and f.match(self.state.intercept) and not f.request.is_replay:
if self.state.intercept and f.match(
self.state.intercept) and not f.request.is_replay:
f.intercept(self)
else:
f.reply()
@ -674,7 +677,7 @@ class ConsoleMaster(flow.FlowMaster):
self.eventlist.append(e)
if len(self.eventlist) > EVENTLOG_SIZE:
self.eventlist.pop(0)
self.eventlist.set_focus(len(self.eventlist)-1)
self.eventlist.set_focus(len(self.eventlist) - 1)
# Handlers
def handle_error(self, f):

6
libmproxy/console/common.py
View File

@ -164,7 +164,7 @@ def raw_format_flow(f, focus, extended, padding):
4: "code_400",
5: "code_500",
}
ccol = codes.get(f["resp_code"]/100, "code_other")
ccol = codes.get(f["resp_code"] / 100, "code_other")
resp.append(fcol(SYMBOL_RETURN, ccol))
if f["resp_is_replay"]:
resp.append(fcol(SYMBOL_REPLAY, "replay"))
@ -200,7 +200,7 @@ def save_data(path, data, master, state):
try:
with file(path, "wb") as f:
f.write(data)
except IOError, v:
except IOError as v:
signals.status_message.send(message=v.strerror)
@ -214,7 +214,7 @@ def ask_save_overwite(path, data, master, state):
save_data(path, data, master, state)
signals.status_prompt_onekey.send(
prompt = "'"+path+"' already exists. Overwite?",
prompt = "'" + path + "' already exists. Overwite?",
keys = (
("yes", "y"),
("no", "n"),

23
libmproxy/console/contentview.py
View File

@ -21,12 +21,12 @@ from ..contrib.wbxml.ASCommandResponse import ASCommandResponse
try:
import pyamf
from pyamf import remoting, flex
except ImportError: # pragma nocover
except ImportError: # pragma nocover
pyamf = None
try:
import cssutils
except ImportError: # pragma nocover
except ImportError: # pragma nocover
cssutils = None
else:
cssutils.log.setLevel(logging.CRITICAL)
@ -36,7 +36,7 @@ else:
cssutils.ser.prefs.indentClosingBrace = False
cssutils.ser.prefs.validOnly = False
VIEW_CUTOFF = 1024*50
VIEW_CUTOFF = 1024 * 50
def _view_text(content, total, limit):
@ -59,7 +59,7 @@ def trailer(clen, txt, limit):
txt.append(
urwid.Text(
[
("highlight", "... %s of data not shown. Press "%netlib.utils.pretty_size(rem)),
("highlight", "... %s of data not shown. Press " % netlib.utils.pretty_size(rem)),
("key", "f"),
("highlight", " to load all data.")
]
@ -76,7 +76,7 @@ class ViewAuto:
ctype = hdrs.get_first("content-type")
if ctype:
ct = utils.parse_content_type(ctype) if ctype else None
ct = "%s/%s"%(ct[0], ct[1])
ct = "%s/%s" % (ct[0], ct[1])
if ct in content_types_map:
return content_types_map[ct][0](hdrs, content, limit)
elif utils.isXML(content):
@ -227,7 +227,7 @@ class ViewURLEncoded:
lines = utils.urldecode(content)
if lines:
body = common.format_keyvals(
[(k+":", v) for (k, v) in lines],
[(k + ":", v) for (k, v) in lines],
key = "header",
val = "text"
)
@ -304,7 +304,6 @@ if pyamf:
if not envelope:
return None
txt = []
for target, message in iter(envelope):
if isinstance(message, pyamf.remoting.Request):
@ -315,13 +314,13 @@ if pyamf:
else:
txt.append(urwid.Text([
("header", "Response: "),
("text", "%s, code %s"%(target, message.status)),
("text", "%s, code %s" % (target, message.status)),
]))
s = json.dumps(self.unpack(message), indent=4)
txt.extend(_view_text(s[:limit], len(s), limit))
return "AMF v%s"%envelope.amfVersion, txt
return "AMF v%s" % envelope.amfVersion, txt
class ViewJavaScript:
@ -375,7 +374,7 @@ class ViewImage:
return None
parts = [
("Format", str(img.format_description)),
("Size", "%s x %s px"%img.size),
("Size", "%s x %s px" % img.size),
("Mode", str(img.mode)),
]
for i in sorted(img.info.keys()):
@ -401,7 +400,7 @@ class ViewImage:
key = "header",
val = "text"
)
return "%s image"%img.format, fmt
return "%s image" % img.format, fmt
class ViewProtobuf:
@ -526,7 +525,7 @@ def get_content_view(viewmode, hdrItems, content, limit, logfunc, is_request):
decoded = encoding.decode(enc, content)
if decoded:
content = decoded
msg.append("[decoded %s]"%enc)
msg.append("[decoded %s]" % enc)
try:
ret = viewmode(hdrs, content, limit)
# Third-party viewers can fail in unexpected ways...

2
libmproxy/console/flowdetailview.py
View File

@ -34,7 +34,7 @@ def flowdetails(state, flow):
if c:
text.append(urwid.Text([("head", "Server Certificate:")]))
parts = [
["Type", "%s, %s bits"%c.keyinfo],
["Type", "%s, %s bits" % c.keyinfo],
["SHA1 digest", c.digest("sha1")],
["Valid to", str(c.notafter)],
["Valid from", str(c.notbefore)],

8
libmproxy/console/flowlist.py
View File

@ -50,7 +50,7 @@ class EventListBox(urwid.ListBox):
elif key == "G":
self.set_focus(0)
elif key == "g":
self.set_focus(len(self.master.eventlist)-1)
self.set_focus(len(self.master.eventlist) - 1)
return urwid.ListBox.keypress(self, size, key)
@ -76,7 +76,8 @@ class BodyPile(urwid.Pile):
def keypress(self, size, key):
if key == "tab":
self.focus_position = (self.focus_position + 1)%len(self.widget_list)
self.focus_position = (
self.focus_position + 1) % len(self.widget_list)
if self.focus_position == 1:
self.widget_list[1].header = self.active_header
else:
@ -157,7 +158,8 @@ class ConnectionItem(urwid.WidgetWrap):
callback = self.master.server_playback_path
)
def keypress(self, (maxcol,), key):
def keypress(self, xxx_todo_changeme, key):
(maxcol,) = xxx_todo_changeme
key = common.shortcuts(key)
if key == "a":
self.flow.accept_intercept(self.master)

101
libmproxy/console/flowview.py
View File

@ -24,42 +24,42 @@ def _mkhelp():
("e", "edit request/response"),
("f", "load full body data"),
("m", "change body display mode for this entity"),
(None,
common.highlight_key("automatic", "a") +
[("text", ": automatic detection")]
),
(None,
common.highlight_key("hex", "e") +
[("text", ": Hex")]
),
(None,
common.highlight_key("html", "h") +
[("text", ": HTML")]
),
(None,
common.highlight_key("image", "i") +
[("text", ": Image")]
),
(None,
common.highlight_key("javascript", "j") +
[("text", ": JavaScript")]
),
(None,
common.highlight_key("json", "s") +
[("text", ": JSON")]
),
(None,
common.highlight_key("urlencoded", "u") +
[("text", ": URL-encoded data")]
),
(None,
common.highlight_key("raw", "r") +
[("text", ": raw data")]
),
(None,
common.highlight_key("xml", "x") +
[("text", ": XML")]
),
(None,
common.highlight_key("automatic", "a") +
[("text", ": automatic detection")]
),
(None,
common.highlight_key("hex", "e") +
[("text", ": Hex")]
),
(None,
common.highlight_key("html", "h") +
[("text", ": HTML")]
),
(None,
common.highlight_key("image", "i") +
[("text", ": Image")]
),
(None,
common.highlight_key("javascript", "j") +
[("text", ": JavaScript")]
),
(None,
common.highlight_key("json", "s") +
[("text", ": JSON")]
),
(None,
common.highlight_key("urlencoded", "u") +
[("text", ": URL-encoded data")]
),
(None,
common.highlight_key("raw", "r") +
[("text", ": raw data")]
),
(None,
common.highlight_key("xml", "x") +
[("text", ": XML")]
),
("M", "change default body display mode"),
("p", "previous flow"),
("P", "copy response(content/headers) to clipboard"),
@ -123,13 +123,13 @@ class FlowView(tabs.Tabs):
def __init__(self, master, state, flow, tab_offset):
self.master, self.state, self.flow = master, state, flow
tabs.Tabs.__init__(self,
[
(self.tab_request, self.view_request),
(self.tab_response, self.view_response),
(self.tab_details, self.view_details),
],
tab_offset
)
[
(self.tab_request, self.view_request),
(self.tab_response, self.view_response),
(self.tab_details, self.view_details),
],
tab_offset
)
self.show()
self.last_displayed_body = None
signals.flow_change.connect(self.sig_flow_change)
@ -173,7 +173,7 @@ class FlowView(tabs.Tabs):
False
)
if full:
limit = sys.maxint
limit = sys.maxsize
else:
limit = contentview.VIEW_CUTOFF
description, text_objects = cache.get(
@ -197,7 +197,7 @@ class FlowView(tabs.Tabs):
def conn_text(self, conn):
if conn:
txt = common.format_keyvals(
[(h+":", v) for (h, v) in conn.headers.lst],
[(h + ":", v) for (h, v) in conn.headers.lst],
key = "header",
val = "text"
)
@ -217,7 +217,7 @@ class FlowView(tabs.Tabs):
" ",
('heading', "["),
('heading_key', "m"),
('heading', (":%s]"%viewmode.name)),
('heading', (":%s]" % viewmode.name)),
],
align="right"
)
@ -272,8 +272,9 @@ class FlowView(tabs.Tabs):
except ValueError:
return None
import BaseHTTPServer
if BaseHTTPServer.BaseHTTPRequestHandler.responses.has_key(int(code)):
response.msg = BaseHTTPServer.BaseHTTPRequestHandler.responses[int(code)][0]
if int(code) in BaseHTTPServer.BaseHTTPRequestHandler.responses:
response.msg = BaseHTTPServer.BaseHTTPRequestHandler.responses[
int(code)][0]
signals.flow_change.send(self, flow = self.flow)
def set_resp_msg(self, msg):
@ -494,7 +495,7 @@ class FlowView(tabs.Tabs):
elif key == "d":
if self.state.flow_count() == 1:
self.master.view_flowlist()
elif self.state.view.index(self.flow) == len(self.state.view)-1:
elif self.state.view.index(self.flow) == len(self.state.view) - 1:
self.view_prev_flow(self.flow)
else:
self.view_next_flow(self.flow)
@ -615,7 +616,7 @@ class FlowView(tabs.Tabs):
if conn.content:
t = conn.headers["content-type"] or [None]
t = t[0]
if os.environ.has_key("EDITOR") or os.environ.has_key("PAGER"):
if "EDITOR" in os.environ or "PAGER" in os.environ:
self.master.spawn_external_viewer(conn.content, t)
else:
signals.status_message.send(

23
libmproxy/console/grideditor.py
View File

@ -175,6 +175,7 @@ class GridWalker(urwid.ListWalker):
and errors is a set with an entry of each offset in rows that is an
error.
"""
def __init__(self, lst, editor):
self.lst = [(i, set([])) for i in lst]
self.editor = editor
@ -225,7 +226,7 @@ class GridWalker(urwid.ListWalker):
def delete_focus(self):
if self.lst:
del self.lst[self.focus]
self.focus = min(len(self.lst)-1, self.focus)
self.focus = min(len(self.lst) - 1, self.focus)
self._modified()
def _insert(self, pos):
@ -266,14 +267,14 @@ class GridWalker(urwid.ListWalker):
self._modified()
def right(self):
self.focus_col = min(self.focus_col + 1, len(self.editor.columns)-1)
self.focus_col = min(self.focus_col + 1, len(self.editor.columns) - 1)
self._modified()
def tab_next(self):
self.stop_edit()
if self.focus_col < len(self.editor.columns)-1:
if self.focus_col < len(self.editor.columns) - 1:
self.focus_col += 1
elif self.focus != len(self.lst)-1:
elif self.focus != len(self.lst) - 1:
self.focus_col = 0
self.focus += 1
self._modified()
@ -297,14 +298,14 @@ class GridWalker(urwid.ListWalker):
self._modified()
def get_next(self, pos):
if pos+1 >= len(self.lst):
if pos + 1 >= len(self.lst):
return None, None
return GridRow(None, False, self.editor, self.lst[pos+1]), pos+1
return GridRow(None, False, self.editor, self.lst[pos + 1]), pos + 1
def get_prev(self, pos):
if pos-1 < 0:
if pos - 1 < 0:
return None, None
return GridRow(None, False, self.editor, self.lst[pos-1]), pos-1
return GridRow(None, False, self.editor, self.lst[pos - 1]), pos - 1
class GridListBox(urwid.ListBox):
@ -387,7 +388,7 @@ class GridEditor(urwid.WidgetWrap):
d = file(p, "rb").read()
self.walker.set_current_value(d, unescaped)
self.walker._modified()
except IOError, v:
except IOError as v:
return str(v)
def set_subeditor_value(self, val, focus, focus_col):
@ -418,7 +419,7 @@ class GridEditor(urwid.WidgetWrap):
elif key == "G":
self.walker.set_focus(0)
elif key == "g":
self.walker.set_focus(len(self.walker.lst)-1)
self.walker.set_focus(len(self.walker.lst) - 1)
elif key in ["h", "left"]:
self.walker.left()
elif key in ["l", "right"]:
@ -633,7 +634,7 @@ class ScriptEditor(GridEditor):
def is_error(self, col, val):
try:
script.Script.parse_command(val)
except script.ScriptError, v:
except script.ScriptError as v:
return str(v)

25
libmproxy/console/help.py
View File

@ -6,7 +6,7 @@ from . import common, signals
from .. import filt, version
footer = [
("heading", 'mitmproxy v%s '%version.VERSION),
("heading", 'mitmproxy v%s ' % version.VERSION),
('heading_key', "q"), ":back ",
]
@ -33,7 +33,12 @@ class HelpView(urwid.ListBox):
("pg up/down", "page up/down"),
("arrows", "up, down, left, right"),
]
text.extend(common.format_keyvals(keys, key="key", val="text", indent=4))
text.extend(
common.format_keyvals(
keys,
key="key",
val="text",
indent=4))
text.append(urwid.Text([("head", "\n\nGlobal keys:\n")]))
keys = [
@ -52,15 +57,15 @@ class HelpView(urwid.ListBox):
f = []
for i in filt.filt_unary:
f.append(
("~%s"%i.code, i.help)
("~%s" % i.code, i.help)
)
for i in filt.filt_rex:
f.append(
("~%s regex"%i.code, i.help)
("~%s regex" % i.code, i.help)
)
for i in filt.filt_int:
f.append(
("~%s int"%i.code, i.help)
("~%s int" % i.code, i.help)
)
f.sort()
f.extend(
@ -75,7 +80,7 @@ class HelpView(urwid.ListBox):
text.append(
urwid.Text(
[
[
"\n",
("text", " Regexes are Python-style.\n"),
("text", " Regexes can be specified as quoted strings.\n"),
@ -83,13 +88,13 @@ class HelpView(urwid.ListBox):
("text", " Expressions with no operators are regex matches against URL.\n"),
("text", " Default binary operator is &.\n"),
("head", "\n Examples:\n"),
]
]
)
)
examples = [
("google\.com", "Url containing \"google.com"),
("~q ~b test", "Requests where body contains \"test\""),
("!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."),
("google\.com", "Url containing \"google.com"),
("~q ~b test", "Requests where body contains \"test\""),
("!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."),
]
text.extend(
common.format_keyvals(examples, key="key", val="text", indent=4)

1
libmproxy/console/options.py
View File

@ -8,6 +8,7 @@ footer = [
('heading_key', "C"), ":clear all ",
]
def _mkhelp():
text = []
keys = [

2
libmproxy/console/palettes.py
View File

@ -270,7 +270,7 @@ class SolarizedDark(LowDark):
# Status bar & heading
heading = (sol_base2, sol_base01),
heading_key = (sol_blue+",bold", sol_base01),
heading_key = (sol_blue + ",bold", sol_base01),
heading_inactive = (sol_base1, sol_base02),
# Help

2
libmproxy/console/pathedit.py
View File

@ -32,7 +32,7 @@ class _PathCompleter:
files = glob.glob(os.path.join(path, "*"))
prefix = txt
else:
files = glob.glob(path+"*")
files = glob.glob(path + "*")
prefix = os.path.dirname(txt)
prefix = prefix or "./"
for f in files:

6
libmproxy/console/searchable.py
View File

@ -37,7 +37,7 @@ class Searchable(urwid.ListBox):
self.set_focus(0)
self.walker._modified()
elif key == "g":
self.set_focus(len(self.walker)-1)
self.set_focus(len(self.walker) - 1)
self.walker._modified()
else:
return super(self.__class__, self).keypress(size, key)
@ -74,11 +74,11 @@ class Searchable(urwid.ListBox):
return
# Start search at focus + 1
if backwards:
rng = xrange(len(self.body)-1, -1, -1)
rng = xrange(len(self.body) - 1, -1, -1)
else:
rng = xrange(1, len(self.body) + 1)
for i in rng:
off = (self.focus_position + i)%len(self.body)
off = (self.focus_position + i) % len(self.body)
w = self.body[off]
txt = self.get_text(w)
if txt and self.search_term in txt:

18
libmproxy/console/select.py
View File

@ -2,6 +2,7 @@ import urwid
from . import common
class _OptionWidget(urwid.WidgetWrap):
def __init__(self, option, text, shortcut, active, focus):
self.option = option
@ -47,14 +48,14 @@ class OptionWalker(urwid.ListWalker):
return self.options[self.focus].render(True), self.focus
def get_next(self, pos):
if pos >= len(self.options)-1:
if pos >= len(self.options) - 1:
return None, None
return self.options[pos+1].render(False), pos+1
return self.options[pos + 1].render(False), pos + 1
def get_prev(self, pos):
if pos <= 0:
return None, None
return self.options[pos-1].render(False), pos-1
return self.options[pos - 1].render(False), pos - 1
class Heading:
@ -69,6 +70,8 @@ class Heading:
_neg = lambda: False
class Option:
def __init__(self, text, shortcut, getstate=None, activate=None):
self.text = text
@ -77,7 +80,12 @@ class Option:
self.activate = activate or _neg
def render(self, focus):
return _OptionWidget(self, self.text, self.shortcut, self.getstate(), focus)
return _OptionWidget(
self,
self.text,
self.shortcut,
self.getstate(),
focus)
class Select(urwid.ListBox):
@ -92,7 +100,7 @@ class Select(urwid.ListBox):
for i in options:
if hasattr(i, "shortcut") and i.shortcut:
if i.shortcut in self.keymap:
raise ValueError("Duplicate shortcut key: %s"%i.shortcut)
raise ValueError("Duplicate shortcut key: %s" % i.shortcut)
self.keymap[i.shortcut] = i
def keypress(self, size, key):

30
libmproxy/console/statusbar.py
View File

@ -58,7 +58,7 @@ class ActionBar(urwid.WidgetWrap):
mkup = []
for i, e in enumerate(keys):
mkup.extend(common.highlight_key(e[0], e[1]))
if i < len(keys)-1:
if i < len(keys) - 1:
mkup.append(",")
prompt.extend(mkup)
prompt.append(")? ")
@ -136,14 +136,14 @@ class StatusBar(urwid.WidgetWrap):
if self.master.client_playback:
r.append("[")
r.append(("heading_key", "cplayback"))
r.append(":%s to go]"%self.master.client_playback.count())
r.append(":%s to go]" % self.master.client_playback.count())
if self.master.server_playback:
r.append("[")
r.append(("heading_key", "splayback"))
if self.master.nopop:
r.append(":%s in file]"%self.master.server_playback.count())
r.append(":%s in file]" % self.master.server_playback.count())
else:
r.append(":%s to go]"%self.master.server_playback.count())
r.append(":%s to go]" % self.master.server_playback.count())
if self.master.get_ignore_filter():
r.append("[")
r.append(("heading_key", "I"))
@ -155,23 +155,23 @@ class StatusBar(urwid.WidgetWrap):
if self.master.state.intercept_txt:
r.append("[")
r.append(("heading_key", "i"))
r.append(":%s]"%self.master.state.intercept_txt)
r.append(":%s]" % self.master.state.intercept_txt)
if self.master.state.limit_txt:
r.append("[")
r.append(("heading_key", "l"))
r.append(":%s]"%self.master.state.limit_txt)
r.append(":%s]" % self.master.state.limit_txt)
if self.master.stickycookie_txt:
r.append("[")
r.append(("heading_key", "t"))
r.append(":%s]"%self.master.stickycookie_txt)
r.append(":%s]" % self.master.stickycookie_txt)
if self.master.stickyauth_txt:
r.append("[")
r.append(("heading_key", "u"))
r.append(":%s]"%self.master.stickyauth_txt)
r.append(":%s]" % self.master.stickyauth_txt)
if self.master.state.default_body_view.name != "Auto":
r.append("[")
r.append(("heading_key", "M"))
r.append(":%s]"%self.master.state.default_body_view.name)
r.append(":%s]" % self.master.state.default_body_view.name)
opts = []
if self.master.anticache:
@ -196,22 +196,22 @@ class StatusBar(urwid.WidgetWrap):
)
if opts:
r.append("[%s]"%(":".join(opts)))
r.append("[%s]" % (":".join(opts)))
if self.master.server.config.mode in ["reverse", "upstream"]:
dst = self.master.server.config.mode.dst
scheme = "https" if dst[0] else "http"
if dst[1] != dst[0]:
scheme += "2https" if dst[1] else "http"
r.append("[dest:%s]"%utils.unparse_url(scheme, *dst[2:]))
r.append("[dest:%s]" % utils.unparse_url(scheme, *dst[2:]))
if self.master.scripts:
r.append("[")
r.append(("heading_key", "s"))
r.append("cripts:%s]"%len(self.master.scripts))
r.append("cripts:%s]" % len(self.master.scripts))
# r.append("[lt:%0.3f]"%self.master.looptime)
if self.master.stream:
r.append("[W:%s]"%self.master.stream_path)
r.append("[W:%s]" % self.master.stream_path)
return r
@ -222,14 +222,14 @@ class StatusBar(urwid.WidgetWrap):
else:
offset = min(self.master.state.focus + 1, fc)
t = [
('heading', ("[%s/%s]"%(offset, fc)).ljust(9))
('heading', ("[%s/%s]" % (offset, fc)).ljust(9))
]
if self.master.server.bound:
host = self.master.server.address.host
if host == "0.0.0.0":
host = "*"
boundaddr = "[%s:%s]"%(host, self.master.server.address.port)
boundaddr = "[%s:%s]" % (host, self.master.server.address.port)
else:
boundaddr = ""
t.extend(self.get_status())

5
libmproxy/console/tabs.py
View File

@ -1,5 +1,6 @@
import urwid
class Tabs(urwid.WidgetWrap):
def __init__(self, tabs, tab_offset=0):
urwid.WidgetWrap.__init__(self, "")
@ -15,10 +16,10 @@ class Tabs(urwid.WidgetWrap):
def keypress(self, size, key):
if key in ["tab", "l"]:
self.tab_offset = (self.tab_offset + 1)%(len(self.tabs))
self.tab_offset = (self.tab_offset + 1) % (len(self.tabs))
self.show()
elif key == "h":
self.tab_offset = (self.tab_offset - 1)%(len(self.tabs))
self.tab_offset = (self.tab_offset - 1) % (len(self.tabs))
self.show()
return self._w.keypress(size, key)

11
libmproxy/controller.py
View File

@ -1,11 +1,14 @@
from __future__ import absolute_import
import Queue, threading
import Queue
import threading
class DummyReply:
"""
A reply object that does nothing. Useful when we need an object to seem
like it has a channel, and during testing.
"""
def __init__(self):
self.acked = False
@ -19,6 +22,7 @@ class Reply:
This object is used to respond to the message through the return
channel.
"""
def __init__(self, obj):
self.obj = obj
self.q = Queue.Queue()
@ -67,11 +71,13 @@ class Slave(threading.Thread):
Slaves get a channel end-point through which they can send messages to
the master.
"""
def __init__(self, channel, server):
self.channel, self.server = channel, server
self.server.set_channel(channel)
threading.Thread.__init__(self)
self.name = "SlaveThread (%s:%s)" % (self.server.address.host, self.server.address.port)
self.name = "SlaveThread (%s:%s)" % (
self.server.address.host, self.server.address.port)
def run(self):
self.server.serve_forever()
@ -81,6 +87,7 @@ class Master(object):
"""
Masters get and respond to messages from slaves.
"""
def __init__(self, server):
"""
server may be None if no server is needed.

25
libmproxy/dump.py
View File

@ -53,7 +53,7 @@ class Options(object):
def str_response(resp):
r = "%s %s"%(resp.code, resp.msg)
r = "%s %s" % (resp.code, resp.msg)
if resp.is_replay:
r = "[replay] " + r
return r
@ -64,7 +64,7 @@ def str_request(f, showhost):
c = f.client_conn.address.host
else:
c = "[replay]"
r = "%s %s %s"%(c, f.request.method, f.request.pretty_url(showhost))
r = "%s %s %s" % (c, f.request.method, f.request.pretty_url(showhost))
if f.request.stickycookie:
r = "[stickycookie] " + r
return r
@ -102,7 +102,7 @@ class DumpMaster(flow.FlowMaster):
try:
f = file(path, options.outfile[1])
self.start_stream(f, self.filt)
except IOError, v:
except IOError as v:
raise DumpError(v.strerror)
if options.replacements:
@ -140,7 +140,7 @@ class DumpMaster(flow.FlowMaster):
if options.rfile:
try:
self.load_flows_file(options.rfile)
except flow.FlowReadError, v:
except flow.FlowReadError as v:
self.add_event("Flow file corrupted.", "error")
raise DumpError(v)
@ -181,12 +181,18 @@ class DumpMaster(flow.FlowMaster):
if not utils.isBin(content):
try:
jsn = json.loads(content)
print(self.indent(4, json.dumps(jsn, indent=2)), file=self.outfile)
print(
self.indent(
4,
json.dumps(
jsn,
indent=2)),
file=self.outfile)
except ValueError:
print(self.indent(4, content), file=self.outfile)
else:
d = netlib.utils.hexdump(content)
d = "\n".join("%s\t%s %s"%i for i in d)
d = "\n".join("%s\t%s %s" % i for i in d)
print(self.indent(4, d), file=self.outfile)
if self.o.flow_detail >= 2:
print("", file=self.outfile)
@ -208,7 +214,12 @@ class DumpMaster(flow.FlowMaster):
sz = "(content missing)"
else:
sz = netlib.utils.pretty_size(len(f.response.content))
print(" << %s %s" % (str_response(f.response), sz), file=self.outfile)
print(
" << %s %s" %
(str_response(
f.response),
sz),
file=self.outfile)
self._print_message(f.response)
if f.error:

10
libmproxy/encoding.py
View File

@ -3,12 +3,14 @@
"""
from __future__ import absolute_import
import cStringIO
import gzip, zlib
import gzip
import zlib
__ALL__ = ["ENCODINGS"]
ENCODINGS = set(["identity", "gzip", "deflate"])
def decode(e, content):
encoding_map = {
"identity": identity,
@ -19,6 +21,7 @@ def decode(e, content):
return None
return encoding_map[e](content)
def encode(e, content):
encoding_map = {
"identity": identity,
@ -29,6 +32,7 @@ def encode(e, content):
return None
return encoding_map[e](content)
def identity(content):
"""
Returns content unchanged. Identity is the default value of
@ -36,6 +40,7 @@ def identity(content):
"""
return content
def decode_gzip(content):
gfile = gzip.GzipFile(fileobj=cStringIO.StringIO(content))
try:
@ -43,6 +48,7 @@ def decode_gzip(content):
except (IOError, EOFError):
return None
def encode_gzip(content):
s = cStringIO.StringIO()
gf = gzip.GzipFile(fileobj=s, mode='wb')
@ -50,6 +56,7 @@ def encode_gzip(content):
gf.close()
return s.getvalue()
def decode_deflate(content):
"""
Returns decompressed data for DEFLATE. Some servers may respond with
@ -67,6 +74,7 @@ def decode_deflate(content):
except zlib.error:
return None
def encode_deflate(content):
"""
Returns compressed content, always including zlib header and checksum.

80
libmproxy/filt.py
View File

@ -32,16 +32,17 @@
rex Equivalent to ~u rex
"""
from __future__ import absolute_import
import re, sys
import re
import sys
from .contrib import pyparsing as pp
from .protocol.http import decoded
class _Token:
def dump(self, indent=0, fp=sys.stdout):
print >> fp, "\t"*indent, self.__class__.__name__,
print >> fp, "\t" * indent, self.__class__.__name__,
if hasattr(self, "expr"):
print >> fp, "(%s)"%self.expr,
print >> fp, "(%s)" % self.expr,
print >> fp
@ -54,6 +55,7 @@ class _Action(_Token):
class FErr(_Action):
code = "e"
help = "Match error"
def __call__(self, f):
return True if f.error else False
@ -61,6 +63,7 @@ class FErr(_Action):
class FReq(_Action):
code = "q"
help = "Match request with no response"
def __call__(self, f):
if not f.response:
return True
@ -69,6 +72,7 @@ class FReq(_Action):
class FResp(_Action):
code = "s"
help = "Match response"
def __call__(self, f):
return True if f.response else False
@ -79,7 +83,7 @@ class _Rex(_Action):
try:
self.re = re.compile(self.expr)
except:
raise ValueError, "Cannot compile expression."
raise ValueError("Cannot compile expression.")
def _check_content_type(expr, o):
@ -100,6 +104,7 @@ class FAsset(_Action):
"image/.*",
"application/x-shockwave-flash"
]
def __call__(self, f):
if f.response:
for i in self.ASSET_TYPES:
@ -111,6 +116,7 @@ class FAsset(_Action):
class FContentType(_Rex):
code = "t"
help = "Content-type header"
def __call__(self, f):
if _check_content_type(self.expr, f.request):
return True
@ -122,6 +128,7 @@ class FContentType(_Rex):
class FRequestContentType(_Rex):
code = "tq"
help = "Request Content-Type header"
def __call__(self, f):
return _check_content_type(self.expr, f.request)
@ -129,6 +136,7 @@ class FRequestContentType(_Rex):
class FResponseContentType(_Rex):
code = "ts"
help = "Response Content-Type header"
def __call__(self, f):
if f.response:
return _check_content_type(self.expr, f.response)
@ -138,6 +146,7 @@ class FResponseContentType(_Rex):
class FHead(_Rex):
code = "h"
help = "Header"
def __call__(self, f):
if f.request.headers.match_re(self.expr):
return True
@ -149,6 +158,7 @@ class FHead(_Rex):
class FHeadRequest(_Rex):
code = "hq"
help = "Request header"
def __call__(self, f):
if f.request.headers.match_re(self.expr):
return True
@ -157,6 +167,7 @@ class FHeadRequest(_Rex):
class FHeadResponse(_Rex):
code = "hs"
help = "Response header"
def __call__(self, f):
if f.response and f.response.headers.match_re(self.expr):
return True
@ -165,6 +176,7 @@ class FHeadResponse(_Rex):
class FBod(_Rex):
code = "b"
help = "Body"
def __call__(self, f):
if f.request and f.request.content:
with decoded(f.request):
@ -180,6 +192,7 @@ class FBod(_Rex):
class FBodRequest(_Rex):
code = "bq"
help = "Request body"
def __call__(self, f):
if f.request and f.request.content:
with decoded(f.request):
@ -190,6 +203,7 @@ class FBodRequest(_Rex):
class FBodResponse(_Rex):
code = "bs"
help = "Response body"
def __call__(self, f):
if f.response and f.response.content:
with decoded(f.response):
@ -200,6 +214,7 @@ class FBodResponse(_Rex):
class FMethod(_Rex):
code = "m"
help = "Method"
def __call__(self, f):
return bool(re.search(self.expr, f.request.method, re.IGNORECASE))
@ -207,6 +222,7 @@ class FMethod(_Rex):
class FDomain(_Rex):
code = "d"
help = "Domain"
def __call__(self, f):
return bool(re.search(self.expr, f.request.host, re.IGNORECASE))
@ -215,6 +231,7 @@ class FUrl(_Rex):
code = "u"
help = "URL"
# FUrl is special, because it can be "naked".
@classmethod
def make(klass, s, loc, toks):
if len(toks) > 1:
@ -233,6 +250,7 @@ class _Int(_Action):
class FCode(_Int):
code = "c"
help = "HTTP response code"
def __call__(self, f):
if f.response and f.response.code == self.num:
return True
@ -243,9 +261,9 @@ class FAnd(_Token):
self.lst = lst
def dump(self, indent=0, fp=sys.stdout):
print >> fp, "\t"*indent, self.__class__.__name__
print >> fp, "\t" * indent, self.__class__.__name__
for i in self.lst:
i.dump(indent+1, fp)
i.dump(indent + 1, fp)
def __call__(self, f):
return all(i(f) for i in self.lst)
@ -256,9 +274,9 @@ class FOr(_Token):
self.lst = lst
def dump(self, indent=0, fp=sys.stdout):
print >> fp, "\t"*indent, self.__class__.__name__
print >> fp, "\t" * indent, self.__class__.__name__
for i in self.lst:
i.dump(indent+1, fp)
i.dump(indent + 1, fp)
def __call__(self, f):
return any(i(f) for i in self.lst)
@ -269,7 +287,7 @@ class FNot(_Token):
self.itm = itm[0]
def dump(self, indent=0, fp=sys.stdout):
print >> fp, "\t"*indent, self.__class__.__name__
print >> fp, "\t" * indent, self.__class__.__name__
self.itm.dump(indent + 1, fp)
def __call__(self, f):
@ -299,26 +317,28 @@ filt_rex = [
filt_int = [
FCode
]
def _make():
# Order is important - multi-char expressions need to come before narrow
# ones.
parts = []
for klass in filt_unary:
f = pp.Literal("~%s"%klass.code)
f = pp.Literal("~%s" % klass.code)
f.setParseAction(klass.make)
parts.append(f)
simplerex = "".join(c for c in pp.printables if c not in "()~'\"")
simplerex = "".join(c for c in pp.printables if c not in "()~'\"")
rex = pp.Word(simplerex) |\
pp.QuotedString("\"", escChar='\\') |\
pp.QuotedString("'", escChar='\\')
pp.QuotedString("\"", escChar='\\') |\
pp.QuotedString("'", escChar='\\')
for klass in filt_rex:
f = pp.Literal("~%s"%klass.code) + rex.copy()
f = pp.Literal("~%s" % klass.code) + rex.copy()
f.setParseAction(klass.make)
parts.append(f)
for klass in filt_int:
f = pp.Literal("~%s"%klass.code) + pp.Word(pp.nums)
f = pp.Literal("~%s" % klass.code) + pp.Word(pp.nums)
f.setParseAction(klass.make)
parts.append(f)
@ -328,14 +348,20 @@ def _make():
parts.append(f)
atom = pp.MatchFirst(parts)
expr = pp.operatorPrecedence(
atom,
[
(pp.Literal("!").suppress(), 1, pp.opAssoc.RIGHT, lambda x: FNot(*x)),
(pp.Literal("&").suppress(), 2, pp.opAssoc.LEFT, lambda x: FAnd(*x)),
(pp.Literal("|").suppress(), 2, pp.opAssoc.LEFT, lambda x: FOr(*x)),
]
)
expr = pp.operatorPrecedence(atom,
[(pp.Literal("!").suppress(),
1,
pp.opAssoc.RIGHT,
lambda x: FNot(*x)),
(pp.Literal("&").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FAnd(*x)),
(pp.Literal("|").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FOr(*x)),
])
expr = pp.OneOrMore(expr)
return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
bnf = _make()
@ -355,15 +381,15 @@ def parse(s):
help = []
for i in filt_unary:
help.append(
("~%s"%i.code, i.help)
("~%s" % i.code, i.help)
)
for i in filt_rex:
help.append(
("~%s regex"%i.code, i.help)
("~%s regex" % i.code, i.help)
)
for i in filt_int:
help.append(
("~%s int"%i.code, i.help)
("~%s int" % i.code, i.help)
)
help.sort()
help.extend(
@ -373,4 +399,4 @@ help.extend(
("|", "or"),
("(...)", "grouping"),
]
)
)

101
libmproxy/flow.py
View File

@ -162,7 +162,8 @@ class StreamLargeBodies(object):
r.headers, is_request, flow.request.method, code
)
if not (0 <= expected_size <= self.max_size):
r.stream = r.stream or True # r.stream may already be a callable, which we want to preserve.
# r.stream may already be a callable, which we want to preserve.
r.stream = r.stream or True
class ClientPlaybackState:
@ -200,8 +201,16 @@ class ClientPlaybackState:
class ServerPlaybackState:
def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content,
ignore_payload_params, ignore_host):
def __init__(
self,
headers,
flows,
exit,
nopop,
ignore_params,
ignore_content,
ignore_payload_params,
ignore_host):
"""
headers: Case-insensitive list of request headers that should be
included in request-response matching.
@ -459,8 +468,9 @@ class FlowStore(FlowList):
Notifies the state that a flow has been updated.
The flow must be present in the state.
"""
for view in self.views:
view._update(f)
if f in self:
for view in self.views:
view._update(f)
def _remove(self, f):
"""
@ -531,7 +541,8 @@ class State(object):
def flow_count(self):
return len(self.flows)
# TODO: All functions regarding flows that don't cause side-effects should be moved into FlowStore.
# TODO: All functions regarding flows that don't cause side-effects should
# be moved into FlowStore.
def index(self, f):
return self.flows.index(f)
@ -659,7 +670,7 @@ class FlowMaster(controller.Master):
"""
try:
s = script.Script(command, self)
except script.ScriptError, v:
except script.ScriptError as v:
return v.args[0]
self.scripts.append(s)
@ -723,8 +734,17 @@ class FlowMaster(controller.Master):
def stop_client_playback(self):
self.client_playback = None
def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params,
ignore_content, ignore_payload_params, ignore_host):
def start_server_playback(
self,
flows,
kill,
headers,
exit,
nopop,
ignore_params,
ignore_content,
ignore_payload_params,
ignore_host):
"""
flows: List of flows.
kill: Boolean, should we kill requests not part of the replay?
@ -733,9 +753,15 @@ class FlowMaster(controller.Master):
ignore_payload_params: list of content params to ignore in server replay
ignore_host: true if request host should be ignored in server replay
"""
self.server_playback = ServerPlaybackState(headers, flows, exit, nopop,
ignore_params, ignore_content,
ignore_payload_params, ignore_host)
self.server_playback = ServerPlaybackState(
headers,
flows,
exit,
nopop,
ignore_params,
ignore_content,
ignore_payload_params,
ignore_host)
self.kill_nonreplay = kill
def stop_server_playback(self):
@ -785,23 +811,36 @@ class FlowMaster(controller.Master):
this method creates a new artificial and minimalist request also adds it to flowlist
"""
c = ClientConnection.from_state(dict(
address=dict(address=(host, port), use_ipv6=False),
clientcert=None
))
address=dict(address=(host, port), use_ipv6=False),
clientcert=None
))
s = ServerConnection.from_state(dict(
address=dict(address=(host, port), use_ipv6=False),
state=[],
source_address=None, #source_address=dict(address=(host, port), use_ipv6=False),
cert=None,
sni=host,
ssl_established=True
))
f = http.HTTPFlow(c,s);
address=dict(address=(host, port), use_ipv6=False),
state=[],
source_address=None,
# source_address=dict(address=(host, port), use_ipv6=False),
cert=None,
sni=host,
ssl_established=True
))
f = http.HTTPFlow(c, s)
headers = ODictCaseless()
req = http.HTTPRequest("absolute", method, scheme, host, port, path, (1, 1), headers, None,
None, None, None)
req = http.HTTPRequest(
"absolute",
method,
scheme,
host,
port,
path,
(1,
1),
headers,
None,
None,
None,
None)
f.request = req
return self.load_flow(f)
@ -812,7 +851,8 @@ class FlowMaster(controller.Master):
if self.server and self.server.config.mode == "reverse":
f.request.host, f.request.port = self.server.config.mode.dst[2:]
f.request.scheme = "https" if self.server.config.mode.dst[1] else "http"
f.request.scheme = "https" if self.server.config.mode.dst[
1] else "http"
f.reply = controller.DummyReply()
if f.request:
@ -839,7 +879,7 @@ class FlowMaster(controller.Master):
try:
f = file(path, "rb")
freader = FlowReader(f)
except IOError, v:
except IOError as v:
raise FlowReadError(v.strerror)
return self.load_flows(freader)
@ -880,7 +920,8 @@ class FlowMaster(controller.Master):
f.backup()
f.request.is_replay = True
if f.request.content:
f.request.headers["Content-Length"] = [str(len(f.request.content))]
f.request.headers[
"Content-Length"] = [str(len(f.request.content))]
f.response = None
f.error = None
self.process_new_request(f)
@ -1031,7 +1072,7 @@ class FlowReader:
"""
off = 0
try:
while 1:
while True:
data = tnetstring.load(self.fo)
if tuple(data["version"][:2]) != version.IVERSION[:2]:
v = ".".join(str(i) for i in data["version"])
@ -1040,7 +1081,7 @@ class FlowReader:
)
off = self.fo.tell()
yield handle.protocols[data["type"]]["flow"].from_state(data)
except ValueError, v:
except ValueError as v:
# Error is due to EOF
if self.fo.tell() == off and self.fo.read() == '':
return

2
libmproxy/main.py
View File

@ -70,7 +70,7 @@ def get_server(dummy_server, options):
else:
try:
return ProxyServer(options)
except ProxyServerError, v:
except ProxyServerError as v:
print(str(v), file=sys.stderr)
sys.exit(1)

13
libmproxy/onboarding/app.py
View File

@ -45,7 +45,10 @@ class PEM(tornado.web.RequestHandler):
def get(self):
p = os.path.join(self.request.master.server.config.cadir, self.filename)
self.set_header("Content-Type", "application/x-x509-ca-cert")
self.set_header("Content-Disposition", "inline; filename={}".format(self.filename))
self.set_header(
"Content-Disposition",
"inline; filename={}".format(
self.filename))
with open(p, "rb") as f:
self.write(f.read())
@ -59,7 +62,10 @@ class P12(tornado.web.RequestHandler):
def get(self):
p = os.path.join(self.request.master.server.config.cadir, self.filename)
self.set_header("Content-Type", "application/x-pkcs12")
self.set_header("Content-Disposition", "inline; filename={}".format(self.filename))
self.set_header(
"Content-Disposition",
"inline; filename={}".format(
self.filename))
with open(p, "rb") as f:
self.write(f.read())
@ -78,7 +84,6 @@ application = tornado.web.Application(
}
),
],
#debug=True
# debug=True
)
mapp = Adapter(application)

3
libmproxy/platform/linux.py
View File

@ -1,4 +1,5 @@
import socket, struct
import socket
import struct
# Python socket module does not have this constant
SO_ORIGINAL_DST = 80

5
libmproxy/platform/osx.py
View File

@ -21,6 +21,7 @@ class Resolver(object):
peer = csock.getpeername()
stxt = subprocess.check_output(self.STATECMD, stderr=subprocess.STDOUT)
if "sudo: a password is required" in stxt:
raise RuntimeError("Insufficient privileges to access pfctl. "
"See http://mitmproxy.org/doc/transparent/osx.html for details.")
raise RuntimeError(
"Insufficient privileges to access pfctl. "
"See http://mitmproxy.org/doc/transparent/osx.html for details.")
return pf.lookup(peer[0], peer[1], stxt)

76
libmproxy/platform/windows.py
View File

@ -197,9 +197,12 @@ class TransparentProxy(object):
self.driver = WinDivert()
self.driver.register()
self.request_filter = custom_filter or " or ".join(("tcp.DstPort == %d" % p) for p in redirect_ports)
self.request_filter = custom_filter or " or ".join(
("tcp.DstPort == %d" %
p) for p in redirect_ports)
self.request_forward_handle = None
self.request_forward_thread = threading.Thread(target=self.request_forward)
self.request_forward_thread = threading.Thread(
target=self.request_forward)
self.request_forward_thread.daemon = True
self.addr_pid_map = dict()
@ -235,17 +238,25 @@ class TransparentProxy(object):
# Block all ICMP requests (which are sent on Windows by default).
# In layman's terms: If we don't do this, our proxy machine tells the client that it can directly connect to the
# real gateway if they are on the same network.
self.icmp_handle = self.driver.open_handle(filter="icmp", layer=Layer.NETWORK, flags=Flag.DROP)
self.icmp_handle = self.driver.open_handle(
filter="icmp",
layer=Layer.NETWORK,
flags=Flag.DROP)
self.response_handle = self.driver.open_handle(filter=self.response_filter, layer=Layer.NETWORK)
self.response_handle = self.driver.open_handle(
filter=self.response_filter,
layer=Layer.NETWORK)
self.response_thread.start()
if self.mode == "forward" or self.mode == "both":
self.request_forward_handle = self.driver.open_handle(filter=self.request_filter,
layer=Layer.NETWORK_FORWARD)
self.request_forward_handle = self.driver.open_handle(
filter=self.request_filter,
layer=Layer.NETWORK_FORWARD)
self.request_forward_thread.start()
if self.mode == "local" or self.mode == "both":
self.request_local_handle = self.driver.open_handle(filter=self.request_filter, layer=Layer.NETWORK)
self.request_local_handle = self.driver.open_handle(
filter=self.request_filter,
layer=Layer.NETWORK)
self.request_local_thread.start()
def shutdown(self):
@ -266,14 +277,17 @@ class TransparentProxy(object):
try:
raw_packet, metadata = handle.recv()
return self.driver.parse_packet(raw_packet), metadata
except WindowsError, e:
except WindowsError as e:
if e.winerror == 995:
return None, None
else:
raise
def fetch_pids(self):
ret = windll.iphlpapi.GetTcpTable2(byref(self.tcptable2), byref(self.tcptable2_size), 0)
ret = windll.iphlpapi.GetTcpTable2(
byref(
self.tcptable2), byref(
self.tcptable2_size), 0)
if ret == ERROR_INSUFFICIENT_BUFFER:
self.tcptable2 = MIB_TCPTABLE2(self.tcptable2_size.value)
self.fetch_pids()
@ -299,7 +313,8 @@ class TransparentProxy(object):
self.fetch_pids()
# If this fails, we most likely have a connection from an external client to
# a local server on 80/443. In this, case we always want to proxy the request.
# a local server on 80/443. In this, case we always want to proxy
# the request.
pid = self.addr_pid_map.get(client, None)
if pid not in self.trusted_pids:
@ -325,7 +340,8 @@ class TransparentProxy(object):
server = (packet.dst_addr, packet.dst_port)
if client in self.client_server_map:
del self.client_server_map[client] # Force re-add to mark as "newest" entry in the dict.
# Force re-add to mark as "newest" entry in the dict.
del self.client_server_map[client]
while len(self.client_server_map) > self.connection_cache_size:
self.client_server_map.popitem(False)
@ -335,7 +351,8 @@ class TransparentProxy(object):
metadata.direction = Direction.INBOUND
packet = self.driver.update_packet_checksums(packet)
# Use any handle thats on the NETWORK layer - request_local may be unavailable.
# Use any handle thats on the NETWORK layer - request_local may be
# unavailable.
self.response_handle.send((packet.raw, metadata))
def response(self):
@ -361,15 +378,32 @@ class TransparentProxy(object):
if __name__ == "__main__":
parser = configargparse.ArgumentParser(description="Windows Transparent Proxy")
parser.add_argument('--mode', choices=['forward', 'local', 'both'], default="both",
help='redirection operation mode: "forward" to only redirect forwarded packets, '
'"local" to only redirect packets originating from the local machine')
parser = configargparse.ArgumentParser(
description="Windows Transparent Proxy")
parser.add_argument(
'--mode',
choices=[
'forward',
'local',
'both'],
default="both",
help='redirection operation mode: "forward" to only redirect forwarded packets, '
'"local" to only redirect packets originating from the local machine')
group = parser.add_mutually_exclusive_group()
group.add_argument("--redirect-ports", nargs="+", type=int, default=[80, 443], metavar="80",
help="ports that should be forwarded to the proxy")
group.add_argument("--custom-filter", default=None, metavar="WINDIVERT_FILTER",
help="Custom WinDivert interception rule.")
group.add_argument(
"--redirect-ports",
nargs="+",
type=int,
default=[
80,
443],
metavar="80",
help="ports that should be forwarded to the proxy")
group.add_argument(
"--custom-filter",
default=None,
metavar="WINDIVERT_FILTER",
help="Custom WinDivert interception rule.")
parser.add_argument("--proxy-addr", default=False,
help="Proxy Server Address")
parser.add_argument("--proxy-port", type=int, default=8080,
@ -391,4 +425,4 @@ if __name__ == "__main__":
except KeyboardInterrupt:
print(" * Shutting down...")
proxy.shutdown()
print(" * Shut down.")
print(" * Shut down.")

2
libmproxy/protocol/__init__.py
View File

@ -1 +1 @@
from .primitives import *
from .primitives import *

5
libmproxy/protocol/handle.py
View File

@ -6,6 +6,7 @@ protocols = {
'tcp': dict(handler=tcp.TCPHandler)
}
def protocol_handler(protocol):
"""
@type protocol: str
@ -14,4 +15,6 @@ def protocol_handler(protocol):
if protocol in protocols:
return protocols[protocol]["handler"]
raise NotImplementedError("Unknown Protocol: %s" % protocol) # pragma: nocover
raise NotImplementedError(
"Unknown Protocol: %s" %
protocol) # pragma: nocover

101
libmproxy/protocol/http.py
View File

@ -305,7 +305,18 @@ class HTTPRequest(HTTPMessage):
@classmethod
def from_state(cls, state):
f = cls(None, None, None, None, None, None, None, None, None, None, None)
f = cls(
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None)
f.load_state(state)
return f
@ -315,7 +326,12 @@ class HTTPRequest(HTTPMessage):
)
@classmethod
def from_stream(cls, rfile, include_body=True, body_size_limit=None, wfile=None):
def from_stream(
cls,
rfile,
include_body=True,
body_size_limit=None,
wfile=None):
"""
Parse an HTTP request from a file stream
@ -403,7 +419,8 @@ class HTTPRequest(HTTPMessage):
self.host,
self.port)]
# If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header.
# If content is defined (i.e. not None or CONTENT_MISSING), we always
# add a content-length header.
if self.content or self.content == "":
headers["Content-Length"] = [str(len(self.content))]
@ -460,9 +477,9 @@ class HTTPRequest(HTTPMessage):
decode appropriately.
"""
if self.headers["accept-encoding"]:
self.headers["accept-encoding"] = [', '.join(
e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0]
)]
self.headers["accept-encoding"] = [
', '.join(
e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0])]
def update_host_header(self):
"""
@ -489,13 +506,22 @@ class HTTPRequest(HTTPMessage):
Returns an empty ODict if there is no data or the content-type
indicates non-form data.
"""
if self.content and self.headers.in_any("content-type", HDR_FORM_URLENCODED, True):
return odict.ODict(utils.urldecode(self.content))
if self.content and self.headers.in_any(
"content-type",
HDR_FORM_URLENCODED,
True):
return odict.ODict(utils.urldecode(self.content))
return odict.ODict([])
def get_form_multipart(self):
if self.content and self.headers.in_any("content-type", HDR_FORM_MULTIPART, True):
return odict.ODict(utils.multipartdecode(self.headers, self.content))
if self.content and self.headers.in_any(
"content-type",
HDR_FORM_MULTIPART,
True):
return odict.ODict(
utils.multipartdecode(
self.headers,
self.content))
return odict.ODict([])
def set_form_urlencoded(self, odict):
@ -664,8 +690,15 @@ class HTTPResponse(HTTPMessage):
timestamp_end: Timestamp indicating when request transmission ended
"""
def __init__(self, httpversion, code, msg, headers, content, timestamp_start=None,
timestamp_end=None):
def __init__(
self,
httpversion,
code,
msg,
headers,
content,
timestamp_start=None,
timestamp_end=None):
assert isinstance(headers, odict.ODictCaseless) or headers is None
HTTPMessage.__init__(
self,
@ -710,7 +743,12 @@ class HTTPResponse(HTTPMessage):
)
@classmethod
def from_stream(cls, rfile, request_method, include_body=True, body_size_limit=None):
def from_stream(
cls,
rfile,
request_method,
include_body=True,
body_size_limit=None):
"""
Parse an HTTP response from a file stream
"""
@ -760,7 +798,8 @@ class HTTPResponse(HTTPMessage):
if not preserve_transfer_encoding:
del headers['Transfer-Encoding']
# If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header.
# If content is defined (i.e. not None or CONTENT_MISSING), we always
# add a content-length header.
if self.content or self.content == "":
headers["Content-Length"] = [str(len(self.content))]
@ -1008,7 +1047,7 @@ class HTTPHandler(ProtocolHandler):
include_body=False
)
break
except (tcp.NetLibError, http.HttpErrorConnClosed), v:
except (tcp.NetLibError, http.HttpErrorConnClosed) as v:
self.c.log(
"error in server communication: %s" % repr(v),
level="debug"
@ -1079,7 +1118,8 @@ class HTTPHandler(ProtocolHandler):
if request_reply is None or request_reply == KILL:
raise KillSignal()
self.process_server_address(flow) # The inline script may have changed request.host
# The inline script may have changed request.host
self.process_server_address(flow)
if isinstance(request_reply, HTTPResponse):
flow.response = request_reply
@ -1090,7 +1130,9 @@ class HTTPHandler(ProtocolHandler):
# we can safely set it as the final attribute value here.
flow.server_conn = self.c.server_conn
self.c.log("response", "debug", [flow.response._assemble_first_line()])
self.c.log(
"response", "debug", [
flow.response._assemble_first_line()])
response_reply = self.c.channel.ask("response", flow)
if response_reply is None or response_reply == KILL:
raise KillSignal()
@ -1117,7 +1159,8 @@ class HTTPHandler(ProtocolHandler):
}
)
)
if not self.process_connect_request((flow.request.host, flow.request.port)):
if not self.process_connect_request(
(flow.request.host, flow.request.port)):
return False
# If the user has changed the target server on this connection,
@ -1130,7 +1173,7 @@ class HTTPHandler(ProtocolHandler):
http.HttpError,
proxy.ProxyError,
tcp.NetLibError,
), e:
) as e:
self.handle_error(e, flow)
except KillSignal:
self.c.log("Connection killed", "info")
@ -1226,7 +1269,8 @@ class HTTPHandler(ProtocolHandler):
# Determine .scheme, .host and .port attributes
# For absolute-form requests, they are directly given in the request.
# For authority-form requests, we only need to determine the request scheme.
# For relative-form requests, we need to determine host and port as well.
# For relative-form requests, we need to determine host and port as
# well.
if not request.scheme:
request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http"
if not request.host:
@ -1253,8 +1297,8 @@ class HTTPHandler(ProtocolHandler):
flow.server_conn = self.c.server_conn
self.c.establish_server_connection()
self.c.client_conn.send(
('HTTP/%s.%s 200 ' % (request.httpversion[0],request.httpversion[1])) +
'Connection established\r\n' +
('HTTP/%s.%s 200 ' % (request.httpversion[0], request.httpversion[1])) +
'Connection established\r\n' +
'Content-Length: 0\r\n' +
('Proxy-agent: %s\r\n' % self.c.config.server_version) +
'\r\n'
@ -1372,10 +1416,15 @@ class HTTPHandler(ProtocolHandler):
semantics. Returns True, if so.
"""
close_connection = (
http.connection_close(flow.request.httpversion, flow.request.headers) or
http.connection_close(flow.response.httpversion, flow.response.headers) or
http.expected_http_body_size(flow.response.headers, False, flow.request.method,
flow.response.code) == -1)
http.connection_close(
flow.request.httpversion,
flow.request.headers) or http.connection_close(
flow.response.httpversion,
flow.response.headers) or http.expected_http_body_size(
flow.response.headers,
False,
flow.request.method,
flow.response.code) == -1)
if close_connection:
if flow.request.form_in == "authority" and flow.response.code == 200:
# Workaround for

13
libmproxy/protocol/primitives.py
View File

@ -24,6 +24,7 @@ class Error(stateobject.StateObject):
msg: Message describing the error
timestamp: Seconds since the epoch
"""
def __init__(self, msg, timestamp=None):
"""
@type msg: str
@ -59,6 +60,7 @@ class Flow(stateobject.StateObject):
A Flow is a collection of objects representing a single transaction.
This class is usually subclassed for each protocol, e.g. HTTPFlow.
"""
def __init__(self, type, client_conn, server_conn, live=None):
self.type = type
self.id = str(uuid.uuid4())
@ -165,12 +167,12 @@ class Flow(stateobject.StateObject):
master.handle_accept_intercept(self)
class ProtocolHandler(object):
"""
A ProtocolHandler implements an application-layer protocol, e.g. HTTP.
See: libmproxy.protocol.http.HTTPHandler
"""
def __init__(self, c):
self.c = c
"""@type: libmproxy.proxy.server.ConnectionHandler"""
@ -209,13 +211,20 @@ class LiveConnection(object):
interface with a live connection, without exposing the internals
of the ConnectionHandler.
"""
def __init__(self, c):
self.c = c
"""@type: libmproxy.proxy.server.ConnectionHandler"""
self._backup_server_conn = None
"""@type: libmproxy.proxy.connection.ServerConnection"""
def change_server(self, address, ssl=None, sni=None, force=False, persistent_change=False):
def change_server(
self,
address,
ssl=None,
sni=None,
force=False,
persistent_change=False):
"""
Change the server connection to the specified address.
@returns:

3
libmproxy/protocol/tcp.py
View File

@ -79,7 +79,8 @@ class TCPHandler(ProtocolHandler):
),
"info"
)
# Do not use dst.connection.send here, which may raise OpenSSL-specific errors.
# Do not use dst.connection.send here, which may raise
# OpenSSL-specific errors.
dst.send(contents)
else:
# socket.socket.send supports raw bytearrays/memoryviews

72
libmproxy/proxy/config.py
View File

@ -81,16 +81,27 @@ class ProxyConfig:
self.check_tcp = HostMatcher(tcp_hosts)
self.authenticator = authenticator
self.cadir = os.path.expanduser(cadir)
self.certstore = certutils.CertStore.from_store(self.cadir, CONF_BASENAME)
self.certstore = certutils.CertStore.from_store(
self.cadir,
CONF_BASENAME)
for spec, cert in certs:
self.certstore.add_cert_file(spec, cert)
self.certforward = certforward
self.openssl_method_client, self.openssl_options_client = version_to_openssl(ssl_version_client)
self.openssl_method_server, self.openssl_options_server = version_to_openssl(ssl_version_server)
self.openssl_method_client, self.openssl_options_client = version_to_openssl(
ssl_version_client)
self.openssl_method_server, self.openssl_options_server = version_to_openssl(
ssl_version_server)
self.ssl_ports = ssl_ports
sslversion_choices = ("all", "secure", "SSLv2", "SSLv3", "TLSv1", "TLSv1_1", "TLSv1_2")
sslversion_choices = (
"all",
"secure",
"SSLv2",
"SSLv3",
"TLSv1",
"TLSv1_1",
"TLSv1_2")
def version_to_openssl(version):
@ -119,7 +130,8 @@ def process_proxy_options(parser, options):
if options.transparent_proxy:
c += 1
if not platform.resolver:
return parser.error("Transparent mode not supported on this platform.")
return parser.error(
"Transparent mode not supported on this platform.")
mode = "transparent"
if options.socks_proxy:
c += 1
@ -133,28 +145,33 @@ def process_proxy_options(parser, options):
mode = "upstream"
upstream_server = options.upstream_proxy
if c > 1:
return parser.error("Transparent, SOCKS5, reverse and upstream proxy mode "
"are mutually exclusive.")
return parser.error(
"Transparent, SOCKS5, reverse and upstream proxy mode "
"are mutually exclusive.")
if options.clientcerts:
options.clientcerts = os.path.expanduser(options.clientcerts)
if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts):
if not os.path.exists(
options.clientcerts) or not os.path.isdir(
options.clientcerts):
return parser.error(
"Client certificate directory does not exist or is not a directory: %s" % options.clientcerts
)
"Client certificate directory does not exist or is not a directory: %s" %
options.clientcerts)
if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd):
if options.auth_singleuser:
if len(options.auth_singleuser.split(':')) != 2:
return parser.error("Invalid single-user specification. Please use the format username:password")
return parser.error(
"Invalid single-user specification. Please use the format username:password")
username, password = options.auth_singleuser.split(':')
password_manager = http_auth.PassManSingleUser(username, password)
elif options.auth_nonanonymous:
password_manager = http_auth.PassManNonAnon()
elif options.auth_htpasswd:
try:
password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd)
except ValueError, v:
password_manager = http_auth.PassManHtpasswd(
options.auth_htpasswd)
except ValueError as v:
return parser.error(v.message)
authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")
else:
@ -203,15 +220,18 @@ def process_proxy_options(parser, options):
def ssl_option_group(parser):
group = parser.add_argument_group("SSL")
group.add_argument(
"--cert", dest='certs', default=[], type=str,
metavar="SPEC", action="append",
"--cert",
dest='certs',
default=[],
type=str,
metavar="SPEC",
action="append",
help='Add an SSL certificate. SPEC is of the form "[domain=]path". '
'The domain may include a wildcard, and is equal to "*" if not specified. '
'The file at path is a certificate in PEM format. If a private key is included in the PEM, '
'it is used, else the default key in the conf dir is used. '
'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. '
'Can be passed multiple times.'
)
'The domain may include a wildcard, and is equal to "*" if not specified. '
'The file at path is a certificate in PEM format. If a private key is included in the PEM, '
'it is used, else the default key in the conf dir is used. '
'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. '
'Can be passed multiple times.')
group.add_argument(
"--cert-forward", action="store_true",
dest="certforward", default=False,
@ -238,11 +258,15 @@ def ssl_option_group(parser):
help="Don't connect to upstream server to look up certificate details."
)
group.add_argument(
"--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS),
"--ssl-port",
action="append",
type=int,
dest="ssl_ports",
default=list(TRANSPARENT_SSL_PORTS),
metavar="PORT",
help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. "
"Defaults to %s." % str(TRANSPARENT_SSL_PORTS)
)
"Defaults to %s." %
str(TRANSPARENT_SSL_PORTS))
group.add_argument(
"--ssl-version-client", dest="ssl_version_client",
default="secure", action="store",

32
libmproxy/proxy/connection.py
View File

@ -7,7 +7,9 @@ from .. import stateobject, utils
class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
def __init__(self, client_connection, address, server):
if client_connection: # Eventually, this object is restored from state. We don't have a connection then.
# Eventually, this object is restored from state. We don't have a
# connection then.
if client_connection:
tcp.BaseHandler.__init__(self, client_connection, address, server)
else:
self.connection = None
@ -39,15 +41,18 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
def get_state(self, short=False):
d = super(ClientConnection, self).get_state(short)
d.update(
address={"address": self.address(), "use_ipv6": self.address.use_ipv6},
clientcert=self.cert.to_pem() if self.clientcert else None
)
address={
"address": self.address(),
"use_ipv6": self.address.use_ipv6},
clientcert=self.cert.to_pem() if self.clientcert else None)
return d
def load_state(self, state):
super(ClientConnection, self).load_state(state)
self.address = tcp.Address(**state["address"]) if state["address"] else None
self.clientcert = certutils.SSLCert.from_pem(state["clientcert"]) if state["clientcert"] else None
self.address = tcp.Address(
**state["address"]) if state["address"] else None
self.clientcert = certutils.SSLCert.from_pem(
state["clientcert"]) if state["clientcert"] else None
def copy(self):
return copy.copy(self)
@ -114,7 +119,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
address={"address": self.address(),
"use_ipv6": self.address.use_ipv6},
source_address= ({"address": self.source_address(),
"use_ipv6": self.source_address.use_ipv6} if self.source_address else None),
"use_ipv6": self.source_address.use_ipv6} if self.source_address else None),
cert=self.cert.to_pem() if self.cert else None
)
return d
@ -122,9 +127,12 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
def load_state(self, state):
super(ServerConnection, self).load_state(state)
self.address = tcp.Address(**state["address"]) if state["address"] else None
self.source_address = tcp.Address(**state["source_address"]) if state["source_address"] else None
self.cert = certutils.SSLCert.from_pem(state["cert"]) if state["cert"] else None
self.address = tcp.Address(
**state["address"]) if state["address"] else None
self.source_address = tcp.Address(
**state["source_address"]) if state["source_address"] else None
self.cert = certutils.SSLCert.from_pem(
state["cert"]) if state["cert"] else None
@classmethod
def from_state(cls, state):
@ -147,7 +155,9 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
def establish_ssl(self, clientcerts, sni, **kwargs):
clientcert = None
if clientcerts:
path = os.path.join(clientcerts, self.address.host.encode("idna")) + ".pem"
path = os.path.join(
clientcerts,
self.address.host.encode("idna")) + ".pem"
if os.path.exists(path):
clientcert = path
self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs)

15
libmproxy/proxy/primitives.py
View File

@ -1,6 +1,7 @@
from __future__ import absolute_import
from netlib import socks
class ProxyError(Exception):
def __init__(self, code, message, headers=None):
super(ProxyError, self).__init__(message)
@ -61,7 +62,7 @@ class TransparentProxyMode(ProxyMode):
def get_upstream_server(self, client_conn):
try:
dst = self.resolver.original_addr(client_conn.connection)
except Exception, e:
except Exception as e:
raise ProxyError(502, "Transparent mode failure: %s" % str(e))
if dst[1] in self.sslports:
@ -87,7 +88,9 @@ class Socks5ProxyMode(ProxyMode):
guess = ""
raise socks.SocksError(
socks.REP.GENERAL_SOCKS_SERVER_FAILURE,
guess + "Invalid SOCKS version. Expected 0x05, got 0x%x" % msg.ver)
guess +
"Invalid SOCKS version. Expected 0x05, got 0x%x" %
msg.ver)
def get_upstream_server(self, client_conn):
try:
@ -117,13 +120,15 @@ class Socks5ProxyMode(ProxyMode):
"mitmproxy only supports SOCKS5 CONNECT."
)
# We do not connect here yet, as the clientconnect event has not been handled yet.
# We do not connect here yet, as the clientconnect event has not
# been handled yet.
connect_reply = socks.Message(
socks.VERSION.SOCKS5,
socks.REP.SUCCEEDED,
socks.ATYP.DOMAINNAME,
client_conn.address # dummy value, we don't have an upstream connection yet.
# dummy value, we don't have an upstream connection yet.
client_conn.address
)
connect_reply.to_file(client_conn.wfile)
client_conn.wfile.flush()
@ -161,4 +166,4 @@ class UpstreamProxyMode(_ConstDestinationProxyMode):
class Log:
def __init__(self, msg, level="info"):
self.msg = msg
self.level = level
self.level = level

95
libmproxy/proxy/server.py
View File

@ -34,7 +34,7 @@ class ProxyServer(tcp.TCPServer):
self.config = config
try:
tcp.TCPServer.__init__(self, (config.host, config.port))
except socket.error, v:
except socket.error as v:
raise ProxyServerError('Error starting proxy server: ' + repr(v))
self.channel = None
@ -46,16 +46,30 @@ class ProxyServer(tcp.TCPServer):
self.channel = channel
def handle_client_connection(self, conn, client_address):
h = ConnectionHandler(self.config, conn, client_address, self, self.channel)
h = ConnectionHandler(
self.config,
conn,
client_address,
self,
self.channel)
h.handle()
h.finish()
class ConnectionHandler:
def __init__(self, config, client_connection, client_address, server, channel):
def __init__(
self,
config,
client_connection,
client_address,
server,
channel):
self.config = config
"""@type: libmproxy.proxy.config.ProxyConfig"""
self.client_conn = ClientConnection(client_connection, client_address, server)
self.client_conn = ClientConnection(
client_connection,
client_address,
server)
"""@type: libmproxy.proxy.connection.ClientConnection"""
self.server_conn = None
"""@type: libmproxy.proxy.connection.ServerConnection"""
@ -70,17 +84,23 @@ class ConnectionHandler:
# Can we already identify the target server and connect to it?
client_ssl, server_ssl = False, False
conn_kwargs = dict()
upstream_info = self.config.mode.get_upstream_server(self.client_conn)
upstream_info = self.config.mode.get_upstream_server(
self.client_conn)
if upstream_info:
self.set_server_address(upstream_info[2:])
client_ssl, server_ssl = upstream_info[:2]
if self.config.check_ignore(self.server_conn.address):
self.log("Ignore host: %s:%s" % self.server_conn.address(), "info")
self.log(
"Ignore host: %s:%s" %
self.server_conn.address(),
"info")
self.conntype = "tcp"
conn_kwargs["log"] = False
client_ssl, server_ssl = False, False
else:
pass # No upstream info from the metadata: upstream info in the protocol (e.g. HTTP absolute-form)
# No upstream info from the metadata: upstream info in the
# protocol (e.g. HTTP absolute-form)
pass
self.channel.ask("clientconnect", self)
@ -92,11 +112,17 @@ class ConnectionHandler:
self.establish_ssl(client=client_ssl, server=server_ssl)
if self.config.check_tcp(self.server_conn.address):
self.log("Generic TCP mode for host: %s:%s" % self.server_conn.address(), "info")
self.log(
"Generic TCP mode for host: %s:%s" %
self.server_conn.address(),
"info")
self.conntype = "tcp"
# Delegate handling to the protocol handler
protocol_handler(self.conntype)(self, **conn_kwargs).handle_messages()
protocol_handler(
self.conntype)(
self,
**conn_kwargs).handle_messages()
self.log("clientdisconnect", "info")
self.channel.tell("clientdisconnect", self)
@ -104,7 +130,8 @@ class ConnectionHandler:
except ProxyError as e:
protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e)
except Exception:
import traceback, sys
import traceback
import sys
self.log(traceback.format_exc(), "error")
print >> sys.stderr, traceback.format_exc()
@ -112,7 +139,8 @@ class ConnectionHandler:
print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy"
finally:
# Make sure that we close the server connection in any case.
# The client connection is closed by the ProxyServer and does not have be handled here.
# The client connection is closed by the ProxyServer and does not
# have be handled here.
self.del_server_connection()
def del_server_connection(self):
@ -122,8 +150,10 @@ class ConnectionHandler:
if self.server_conn and self.server_conn.connection:
self.server_conn.finish()
self.server_conn.close()
self.log("serverdisconnect", "debug", ["%s:%s" % (self.server_conn.address.host,
self.server_conn.address.port)])
self.log(
"serverdisconnect", "debug", [
"%s:%s" %
(self.server_conn.address.host, self.server_conn.address.port)])
self.channel.tell("serverdisconnect", self)
self.server_conn = None
@ -141,7 +171,9 @@ class ConnectionHandler:
if self.server_conn:
self.del_server_connection()
self.log("Set new server address: %s:%s" % (address.host, address.port), "debug")
self.log(
"Set new server address: %s:%s" %
(address.host, address.port), "debug")
self.server_conn = ServerConnection(address)
def establish_server_connection(self, ask=True):
@ -155,12 +187,16 @@ class ConnectionHandler:
"""
if self.server_conn.connection:
return
self.log("serverconnect", "debug", ["%s:%s" % self.server_conn.address()[:2]])
self.log(
"serverconnect", "debug", [
"%s:%s" %
self.server_conn.address()[
:2]])
if ask:
self.channel.ask("serverconnect", self)
try:
self.server_conn.connect()
except tcp.NetLibError, v:
except tcp.NetLibError as v:
raise ProxyError(502, v)
def establish_ssl(self, client=False, server=False, sni=None):
@ -237,7 +273,8 @@ class ConnectionHandler:
self.server_conn.state = state
# Receiving new_sni where had_ssl is False is a weird case that happens when the workaround for
# https://github.com/mitmproxy/mitmproxy/issues/427 is active. In this case, we want to establish SSL as well.
# https://github.com/mitmproxy/mitmproxy/issues/427 is active. In this
# case, we want to establish SSL as well.
if had_ssl or new_sni:
self.establish_ssl(server=True, sni=sni)
@ -246,8 +283,10 @@ class ConnectionHandler:
def log(self, msg, level, subs=()):
msg = [
"%s:%s: %s" % (self.client_conn.address.host, self.client_conn.address.port, msg)
]
"%s:%s: %s" %
(self.client_conn.address.host,
self.client_conn.address.port,
msg)]
for i in subs:
msg.append(" -> " + i)
msg = "\n".join(msg)
@ -255,11 +294,13 @@ class ConnectionHandler:
def find_cert(self):
if self.config.certforward and self.server_conn.ssl_established:
return self.server_conn.cert, self.config.certstore.gen_pkey(self.server_conn.cert), None
return self.server_conn.cert, self.config.certstore.gen_pkey(
self.server_conn.cert), None
else:
host = self.server_conn.address.host
sans = []
if self.server_conn.ssl_established and (not self.config.no_upstream_cert):
if self.server_conn.ssl_established and (
not self.config.no_upstream_cert):
upstream_cert = self.server_conn.cert
sans.extend(upstream_cert.altnames)
if upstream_cert.cn:
@ -291,8 +332,11 @@ class ConnectionHandler:
# - We established SSL with the server previously
# - We initially wanted to establish SSL with the server,
# but the server refused to negotiate without SNI.
if self.server_conn.ssl_established or hasattr(self.server_conn, "may_require_sni"):
self.server_reconnect(sni) # reconnect to upstream server with SNI
if self.server_conn.ssl_established or hasattr(
self.server_conn,
"may_require_sni"):
# reconnect to upstream server with SNI
self.server_reconnect(sni)
# Now, change client context to reflect changed certificate:
cert, key, chain_file = self.find_cert()
new_context = self.client_conn.create_ssl_context(
@ -308,4 +352,7 @@ class ConnectionHandler:
# make dang sure it doesn't happen.
except: # pragma: no cover
import traceback
self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error")
self.log(
"Error in handle_sni:\r\n" +
traceback.format_exc(),
"error")

35
libmproxy/script.py
View File

@ -1,7 +1,11 @@
from __future__ import absolute_import
import os, traceback, threading, shlex
import os
import traceback
import threading
import shlex
from . import controller
class ScriptError(Exception):
pass
@ -56,6 +60,7 @@ class Script:
s = Script(argv, master)
s.load()
"""
def __init__(self, command, master):
self.command = command
self.argv = self.parse_command(command)
@ -73,9 +78,11 @@ class Script:
args = shlex.split(command)
args[0] = os.path.expanduser(args[0])
if not os.path.exists(args[0]):
raise ScriptError(("Script file not found: %s.\r\n"
"If you script path contains spaces, "
"make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") % args[0])
raise ScriptError(
("Script file not found: %s.\r\n"
"If you script path contains spaces, "
"make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") %
args[0])
elif not os.path.isfile(args[0]):
raise ScriptError("Not a file: %s" % args[0])
return args
@ -90,7 +97,7 @@ class Script:
ns = {}
try:
execfile(self.argv[0], ns, ns)
except Exception, v:
except Exception as v:
raise ScriptError(traceback.format_exc(v))
self.ns = ns
r = self.run("start", self.argv)
@ -114,7 +121,7 @@ class Script:
if f:
try:
return (True, f(self.ctx, *args, **kwargs))
except Exception, v:
except Exception as v:
return (False, (v, traceback.format_exc(v)))
else:
return (False, None)
@ -133,7 +140,7 @@ class ReplyProxy(object):
return
self.original_reply(*args, **kwargs)
def __getattr__ (self, k):
def __getattr__(self, k):
return getattr(self.original_reply, k)
@ -145,7 +152,8 @@ def _handle_concurrent_reply(fn, o, *args, **kwargs):
def run():
fn(*args, **kwargs)
reply_proxy() # If the script did not call .reply(), we have to do it now.
# If the script did not call .reply(), we have to do it now.
reply_proxy()
ScriptThread(target=run).start()
@ -154,8 +162,15 @@ class ScriptThread(threading.Thread):
def concurrent(fn):
if fn.func_name in ("request", "response", "error", "clientconnect", "serverconnect", "clientdisconnect"):
if fn.func_name in (
"request",
"response",
"error",
"clientconnect",
"serverconnect",
"clientdisconnect"):
def _concurrent(ctx, obj):
_handle_concurrent_reply(fn, obj, ctx, obj)
return _concurrent
raise NotImplementedError("Concurrent decorator not supported for this method.")
raise NotImplementedError(
"Concurrent decorator not supported for this method.")

92
libmproxy/tnetstring.py
View File

@ -72,13 +72,14 @@ __ver_major__ = 0
__ver_minor__ = 2
__ver_patch__ = 0
__ver_sub__ = ""
__version__ = "%d.%d.%d%s" % (__ver_major__,__ver_minor__,__ver_patch__,__ver_sub__)
__version__ = "%d.%d.%d%s" % (
__ver_major__, __ver_minor__, __ver_patch__, __ver_sub__)
from collections import deque
def dumps(value,encoding=None):
def dumps(value, encoding=None):
"""dumps(object,encoding=None) -> string
This function dumps a python object as a tnetstring.
@ -90,21 +91,21 @@ def dumps(value,encoding=None):
# consider the _gdumps() function instead; it's a standard top-down
# generator that's simpler to understand but much less efficient.
q = deque()
_rdumpq(q,0,value,encoding)
_rdumpq(q, 0, value, encoding)
return "".join(q)
def dump(value,file,encoding=None):
def dump(value, file, encoding=None):
"""dump(object,file,encoding=None)
This function dumps a python object as a tnetstring and writes it to
the given file.
"""
file.write(dumps(value,encoding))
file.write(dumps(value, encoding))
file.flush()
def _rdumpq(q,size,value,encoding=None):
def _rdumpq(q, size, value, encoding=None):
"""Dump value as a tnetstring, to a deque instance, last chunks first.
This function generates the tnetstring representation of the given value,
@ -129,7 +130,7 @@ def _rdumpq(q,size,value,encoding=None):
if value is False:
write("5:false!")
return size + 8
if isinstance(value,(int,long)):
if isinstance(value, (int, long)):
data = str(value)
ldata = len(data)
span = str(ldata)
@ -138,7 +139,7 @@ def _rdumpq(q,size,value,encoding=None):
write(":")
write(span)
return size + 2 + len(span) + ldata
if isinstance(value,(float,)):
if isinstance(value, (float,)):
# Use repr() for float rather than str().
# It round-trips more accurately.
# Probably unnecessary in later python versions that
@ -151,7 +152,7 @@ def _rdumpq(q,size,value,encoding=None):
write(":")
write(span)
return size + 2 + len(span) + ldata
if isinstance(value,str):
if isinstance(value, str):
lvalue = len(value)
span = str(lvalue)
write(",")
@ -159,26 +160,26 @@ def _rdumpq(q,size,value,encoding=None):
write(":")
write(span)
return size + 2 + len(span) + lvalue
if isinstance(value,(list,tuple,)):
if isinstance(value, (list, tuple,)):
write("]")
init_size = size = size + 1
for item in reversed(value):
size = _rdumpq(q,size,item,encoding)
size = _rdumpq(q, size, item, encoding)
span = str(size - init_size)
write(":")
write(span)
return size + 1 + len(span)
if isinstance(value,dict):
if isinstance(value, dict):
write("}")
init_size = size = size + 1
for (k,v) in value.iteritems():
size = _rdumpq(q,size,v,encoding)
size = _rdumpq(q,size,k,encoding)
for (k, v) in value.iteritems():
size = _rdumpq(q, size, v, encoding)
size = _rdumpq(q, size, k, encoding)
span = str(size - init_size)
write(":")
write(span)
return size + 1 + len(span)
if isinstance(value,unicode):
if isinstance(value, unicode):
if encoding is None:
raise ValueError("must specify encoding to dump unicode strings")
value = value.encode(encoding)
@ -192,7 +193,7 @@ def _rdumpq(q,size,value,encoding=None):
raise ValueError("unserializable object")
def _gdumps(value,encoding):
def _gdumps(value, encoding):
"""Generate fragments of value dumped as a tnetstring.
This is the naive dumping algorithm, implemented as a generator so that
@ -207,24 +208,24 @@ def _gdumps(value,encoding):
yield "4:true!"
elif value is False:
yield "5:false!"
elif isinstance(value,(int,long)):
elif isinstance(value, (int, long)):
data = str(value)
yield str(len(data))
yield ":"
yield data
yield "#"
elif isinstance(value,(float,)):
elif isinstance(value, (float,)):
data = repr(value)
yield str(len(data))
yield ":"
yield data
yield "^"
elif isinstance(value,(str,)):
elif isinstance(value, (str,)):
yield str(len(value))
yield ":"
yield value
yield ","
elif isinstance(value,(list,tuple,)):
elif isinstance(value, (list, tuple,)):
sub = []
for item in value:
sub.extend(_gdumps(item))
@ -233,9 +234,9 @@ def _gdumps(value,encoding):
yield ":"
yield sub
yield "]"
elif isinstance(value,(dict,)):
elif isinstance(value, (dict,)):
sub = []
for (k,v) in value.iteritems():
for (k, v) in value.iteritems():
sub.extend(_gdumps(k))
sub.extend(_gdumps(v))
sub = "".join(sub)
@ -243,7 +244,7 @@ def _gdumps(value,encoding):
yield ":"
yield sub
yield "}"
elif isinstance(value,(unicode,)):
elif isinstance(value, (unicode,)):
if encoding is None:
raise ValueError("must specify encoding to dump unicode strings")
value = value.encode(encoding)
@ -255,7 +256,7 @@ def _gdumps(value,encoding):
raise ValueError("unserializable object")
def loads(string,encoding=None):
def loads(string, encoding=None):
"""loads(string,encoding=None) -> object
This function parses a tnetstring into a python object.
@ -263,10 +264,10 @@ def loads(string,encoding=None):
# No point duplicating effort here. In the C-extension version,
# loads() is measurably faster then pop() since it can avoid
# the overhead of building a second string.
return pop(string,encoding)[0]
return pop(string, encoding)[0]
def load(file,encoding=None):
def load(file, encoding=None):
"""load(file,encoding=None) -> object
This function reads a tnetstring from a file and parses it into a
@ -324,21 +325,20 @@ def load(file,encoding=None):
if type == "]":
l = []
while data:
(item,data) = pop(data,encoding)
(item, data) = pop(data, encoding)
l.append(item)
return l
if type == "}":
d = {}
while data:
(key,data) = pop(data,encoding)
(val,data) = pop(data,encoding)
(key, data) = pop(data, encoding)
(val, data) = pop(data, encoding)
d[key] = val
return d
raise ValueError("unknown type tag")
def pop(string,encoding=None):
def pop(string, encoding=None):
"""pop(string,encoding=None) -> (object, remain)
This function parses a tnetstring into a python object.
@ -347,12 +347,12 @@ def pop(string,encoding=None):
"""
# Parse out data length, type and remaining string.
try:
(dlen,rest) = string.split(":",1)
(dlen, rest) = string.split(":", 1)
dlen = int(dlen)
except ValueError:
raise ValueError("not a tnetstring: missing or invalid length prefix")
try:
(data,type,remain) = (rest[:dlen],rest[dlen],rest[dlen+1:])
(data, type, remain) = (rest[:dlen], rest[dlen], rest[dlen + 1:])
except IndexError:
# This fires if len(rest) < dlen, meaning we don't need
# to further validate that data is the right length.
@ -360,40 +360,40 @@ def pop(string,encoding=None):
# Parse the data based on the type tag.
if type == ",":
if encoding is not None:
return (data.decode(encoding),remain)
return (data,remain)
return (data.decode(encoding), remain)
return (data, remain)
if type == "#":
try:
return (int(data),remain)
return (int(data), remain)
except ValueError:
raise ValueError("not a tnetstring: invalid integer literal")
if type == "^":
try:
return (float(data),remain)
return (float(data), remain)
except ValueError:
raise ValueError("not a tnetstring: invalid float literal")
if type == "!":
if data == "true":
return (True,remain)
return (True, remain)
elif data == "false":
return (False,remain)
return (False, remain)
else:
raise ValueError("not a tnetstring: invalid boolean literal")
if type == "~":
if data:
raise ValueError("not a tnetstring: invalid null literal")
return (None,remain)
return (None, remain)
if type == "]":
l = []
while data:
(item,data) = pop(data,encoding)
(item, data) = pop(data, encoding)
l.append(item)
return (l,remain)
return (l, remain)
if type == "}":
d = {}
while data:
(key,data) = pop(data,encoding)
(val,data) = pop(data,encoding)
(key, data) = pop(data, encoding)
(val, data) = pop(data, encoding)
d[key] = val
return (d,remain)
return (d, remain)
raise ValueError("unknown type tag")

29
libmproxy/utils.py
View File

@ -1,8 +1,14 @@
from __future__ import absolute_import
import os, datetime, urllib, re
import time, functools, cgi
import os
import datetime
import urllib
import re
import time
import functools
import cgi
import json
def timestamp():
"""
Returns a serializable UTC timestamp.
@ -91,7 +97,7 @@ def multipartdecode(hdrs, content):
match = rx.search(parts[1])
if match:
key = match.group(1)
value = "".join(parts[3+parts[2:].index(""):])
value = "".join(parts[3 + parts[2:].index(""):])
r.append((key, value))
return r
return []
@ -107,8 +113,8 @@ def pretty_duration(secs):
for limit, formatter in formatters:
if secs >= limit:
return formatter.format(secs)
#less than 1 sec
return "{:.0f}ms".format(secs*1000)
# less than 1 sec
return "{:.0f}ms".format(secs * 1000)
class Data:
@ -126,7 +132,7 @@ class Data:
"""
fullpath = os.path.join(self.dirname, path)
if not os.path.exists(fullpath):
raise ValueError, "dataPath: %s does not exist."%fullpath
raise ValueError("dataPath: %s does not exist." % fullpath)
return fullpath
pkg_data = Data(__name__)
@ -135,10 +141,11 @@ class LRUCache:
"""
A simple LRU cache for generated values.
"""
def __init__(self, size=100):
self.size = size
self.cache = {}
self.cacheList = []
self.cacheList = []
def get(self, gen, *args):
"""
@ -147,7 +154,7 @@ class LRUCache:
*args: A list of immutable arguments, used to establish identiy by
*the cache, and passed to gen to generate values.
"""
if self.cache.has_key(args):
if args in self.cache:
self.cacheList.remove(args)
self.cacheList.insert(0, args)
return self.cache[args]
@ -195,14 +202,14 @@ def hostport(scheme, host, port):
if (port, scheme) in [(80, "http"), (443, "https")]:
return host
else:
return "%s:%s"%(host, port)
return "%s:%s" % (host, port)
def unparse_url(scheme, host, port, path=""):
"""
Returns a URL string, constructed from the specified compnents.
"""
return "%s://%s%s"%(scheme, hostport(scheme, host, port), path)
return "%s://%s%s" % (scheme, hostport(scheme, host, port), path)
def clean_hanging_newline(t):
@ -243,7 +250,7 @@ def parse_size(s):
try:
return int(s) * mult
except ValueError:
raise ValueError("Invalid size specification: %s"%s)
raise ValueError("Invalid size specification: %s" % s)
def safe_subn(pattern, repl, target, *args, **kwargs):

2
libmproxy/version.py
View File

@ -1,4 +1,4 @@
IVERSION = (0, 12, 0)
IVERSION = (0, 12, 1)
VERSION = ".".join(str(i) for i in IVERSION)
MINORVERSION = ".".join(str(i) for i in IVERSION[:2])
NAME = "mitmproxy"

9
libmproxy/web/__init__.py
View File

@ -79,6 +79,7 @@ class WebState(flow.State):
data=[]
)
class Options(object):
attributes = [
"app",
@ -128,15 +129,14 @@ class WebMaster(flow.FlowMaster):
if options.rfile:
try:
self.load_flows_file(options.rfile)
except flow.FlowReadError, v:
except flow.FlowReadError as v:
self.add_event(
"Could not read flow file: %s"%v,
"Could not read flow file: %s" % v,
"error"
)
if self.options.app:
self.start_app(self.options.app_host, self.options.app_port)
def tick(self):
flow.FlowMaster.tick(self, self.masterq, timeout=0)
@ -157,7 +157,8 @@ class WebMaster(flow.FlowMaster):
self.shutdown()
def _process_flow(self, f):
if self.state.intercept and self.state.intercept(f) and not f.request.is_replay:
if self.state.intercept and self.state.intercept(
f) and not f.request.is_replay:
f.intercept(self)
else:
f.reply()

9
libmproxy/web/app.py
View File

@ -27,7 +27,8 @@ class RequestHandler(tornado.web.RequestHandler):
@property
def json(self):
if not self.request.headers.get("Content-Type").startswith("application/json"):
if not self.request.headers.get(
"Content-Type").startswith("application/json"):
return None
return json.loads(self.request.body)
@ -67,8 +68,10 @@ class FiltHelp(RequestHandler):
commands=filt.help
))
class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler):
connections = None # raise an error if inherited class doesn't specify its own instance.
# raise an error if inherited class doesn't specify its own instance.
connections = None
def open(self):
self.connections.add(self)
@ -264,4 +267,4 @@ class Application(tornado.web.Application):
cookie_secret=os.urandom(256),
debug=debug,
)
super(Application, self).__init__(handlers, **settings)
super(Application, self).__init__(handlers, **settings)

2
mitmdump
View File

@ -1,3 +1,3 @@
#!/usr/bin/env python
from libmproxy.main import mitmdump
mitmdump()
mitmdump()

2
mitmproxy
View File

@ -1,3 +1,3 @@
#!/usr/bin/env python
from libmproxy.main import mitmproxy
mitmproxy()
mitmproxy()

4
release/mitmdump.spec
View File

@ -16,6 +16,10 @@ a.datas += Tree(
"./libmproxy/onboarding/static",
prefix="libmproxy/onboarding/static"
)
a.datas += Tree(
"../venv.mitmproxy/lib/python2.7/site-packages/cryptography/hazmat/bindings/openssl/src",
prefix = "cryptography/hazmat/bindings/openssl/src"
)
pyz = PYZ(a.pure)
exe = EXE(pyz,
a.scripts,

4
release/mitmproxy.spec
View File

@ -16,6 +16,10 @@ a.datas += Tree(
"./libmproxy/onboarding/static",
prefix="libmproxy/onboarding/static"
)
a.datas += Tree(
"../venv.mitmproxy/lib/python2.7/site-packages/cryptography/hazmat/bindings/openssl/src",
prefix = "cryptography/hazmat/bindings/openssl/src"
)
pyz = PYZ(a.pure)
exe = EXE(pyz,
a.scripts,

4
release/mitmweb.spec
View File

@ -24,6 +24,10 @@ a.datas += Tree(
"./libmproxy/web/static",
prefix="libmproxy/web/static"
)
a.datas += Tree(
"../venv.mitmproxy/lib/python2.7/site-packages/cryptography/hazmat/bindings/openssl/src",
prefix = "cryptography/hazmat/bindings/openssl/src"
)
pyz = PYZ(a.pure)
exe = EXE(pyz,
a.scripts,

3
release/release-checklist
View File

@ -5,6 +5,9 @@
netlib/netlib/version.py
pathod/libpathod/version.py
- Ensure that the website style assets have been compiled for production, and
synced to the docs.
- Render the docs:
cshape doc-src doc

9
setup.cfg Normal file
View File

@ -0,0 +1,9 @@
[flake8]
max-line-length = 80
max-complexity = 15
[pep8]
max-line-length = 80
max-complexity = 15
exclude = */contrib/*
ignore = E251,E309

22
setup.py
View File

@ -67,13 +67,11 @@ setup(
"Topic :: Internet",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Internet :: Proxy Servers",
"Topic :: Software Development :: Testing"
],
"Topic :: Software Development :: Testing"],
packages=find_packages(),
include_package_data=True,
entry_points={
'console_scripts': console_scripts
},
'console_scripts': console_scripts},
install_requires=list(deps),
extras_require={
'dev': [
@ -81,19 +79,15 @@ setup(
"nose>=1.3.0",
"nose-cov>=1.6",
"coveralls>=0.4.1",
"pathod>=%s, <%s" % (
version.MINORVERSION, version.NEXT_MINORVERSION
)
],
"pathod>=%s, <%s" %
(version.MINORVERSION,
version.NEXT_MINORVERSION),
"countershape"],
'contentviews': [
"pyamf>=0.6.1",
"protobuf>=2.5.0",
"cssutils>=1.0"
],
"cssutils>=1.0"],
'examples': [
"pytz",
"harparser",
"beautifulsoup4"
]
}
)
"beautifulsoup4"]})

6
test/mock_urwid.py
View File

@ -1,8 +1,10 @@
import os, sys, mock
import os
import sys
import mock
if os.name == "nt":
m = mock.Mock()
m.__version__ = "1.1.1"
m.Widget = mock.Mock
m.WidgetWrap = mock.Mock
sys.modules['urwid'] = m
sys.modules['urwid.util'] = mock.Mock()
sys.modules['urwid.util'] = mock.Mock()

4
test/scripts/a.py
View File

@ -4,14 +4,18 @@ parser = argparse.ArgumentParser()
parser.add_argument('--var', type=int)
var = 0
def start(ctx, argv):
global var
var = parser.parse_args(argv[1:]).var
def here(ctx):
global var
var += 1
return var
def errargs():
pass

8
test/scripts/all.py
View File

@ -1,28 +1,36 @@
log = []
def clientconnect(ctx, cc):
ctx.log("XCLIENTCONNECT")
log.append("clientconnect")
def serverconnect(ctx, cc):
ctx.log("XSERVERCONNECT")
log.append("serverconnect")
def request(ctx, f):
ctx.log("XREQUEST")
log.append("request")
def response(ctx, f):
ctx.log("XRESPONSE")
log.append("response")
def responseheaders(ctx, f):
ctx.log("XRESPONSEHEADERS")
log.append("responseheaders")
def clientdisconnect(ctx, cc):
ctx.log("XCLIENTDISCONNECT")
log.append("clientdisconnect")
def error(ctx, cc):
ctx.log("XERROR")
log.append("error")

2
test/scripts/concurrent_decorator.py
View File

@ -29,4 +29,4 @@ def error(context, err):
@concurrent
def clientdisconnect(context, dc):
context.log("clientdisconnect")
context.log("clientdisconnect")

3
test/scripts/concurrent_decorator_err.py
View File

@ -1,5 +1,6 @@
from libmproxy.script import concurrent
@concurrent
def start(context, argv):
pass
pass

1
test/scripts/duplicate_flow.py
View File

@ -2,4 +2,3 @@
def request(ctx, f):
f = ctx.duplicate_flow(f)
ctx.replay_request(f)

2
test/scripts/stream_modify.py
View File

@ -4,4 +4,4 @@ def modify(chunks):
def responseheaders(context, flow):
flow.response.stream = modify
flow.response.stream = modify

9
test/test_app.py
View File

@ -1,8 +1,13 @@
import mock, socket, os, time
import mock
import socket
import os
import time
from libmproxy import dump
from netlib import certutils, tcp
from libpathod.pathoc import Pathoc
import tutils, tservers
import tutils
import tservers
class TestApp(tservers.HTTPProxTest):
def test_basic(self):

29
test/test_cmdline.py
View File

@ -37,13 +37,24 @@ def test_parse_replace_hook():
def test_parse_server_spec():
tutils.raises("Invalid server specification", cmdline.parse_server_spec, "")
assert cmdline.parse_server_spec("http://foo.com:88") == [False, False, "foo.com", 88]
assert cmdline.parse_server_spec("http://foo.com") == [False, False, "foo.com", 80]
assert cmdline.parse_server_spec("https://foo.com") == [True, True, "foo.com", 443]
assert cmdline.parse_server_spec_special("https2http://foo.com") == [True, False, "foo.com", 80]
assert cmdline.parse_server_spec_special("http2https://foo.com") == [False, True, "foo.com", 443]
tutils.raises("Invalid server specification", cmdline.parse_server_spec, "foo.com")
tutils.raises("Invalid server specification", cmdline.parse_server_spec, "http://")
assert cmdline.parse_server_spec(
"http://foo.com:88") == [False, False, "foo.com", 88]
assert cmdline.parse_server_spec(
"http://foo.com") == [False, False, "foo.com", 80]
assert cmdline.parse_server_spec(
"https://foo.com") == [True, True, "foo.com", 443]
assert cmdline.parse_server_spec_special(
"https2http://foo.com") == [True, False, "foo.com", 80]
assert cmdline.parse_server_spec_special(
"http2https://foo.com") == [False, True, "foo.com", 443]
tutils.raises(
"Invalid server specification",
cmdline.parse_server_spec,
"foo.com")
tutils.raises(
"Invalid server specification",
cmdline.parse_server_spec,
"http://")
def test_parse_setheaders():
@ -103,7 +114,7 @@ def test_common():
)
p = tutils.test_data.path("data/replace")
opts.replace_file = [("/foo/bar/%s"%p)]
opts.replace_file = [("/foo/bar/%s" % p)]
v = cmdline.get_common_options(opts)["replacements"]
assert len(v) == 1
assert v[0][2].strip() == "replacecontents"
@ -122,5 +133,3 @@ def test_mitmdump():
def test_mitmweb():
ap = cmdline.mitmweb()
assert ap

Some files were not shown because too many files have changed in this diff Show More