diff --git a/CHANGELOG b/CHANGELOG
index 86c41b13a..e0fb71b6e 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,10 @@
+29 Dec 2014: mitmproxy 0.11.2:
+
+ * Configuration files - mitmproxy.conf, mitmdump.conf, common.conf in the
+ .mitmproxy directory.
+ * Better handling of servers that reject connections that are not SNI.
+ * Many other small bugfixes and improvements.
+
15 November 2014: mitmproxy 0.11.1:
diff --git a/MANIFEST.in b/MANIFEST.in
index cc048b614..3578d855e 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -5,4 +5,6 @@ recursive-include examples *
recursive-include doc *
recursive-include test *
recursive-include libmproxy *
-recursive-exclude * *.pyc *.pyo *.swo *.swp
\ No newline at end of file
+recursive-exclude * *.pyc *.pyo *.swo *.swp
+recursive-exclude netlib *
+recursive-exclude libpathod *
diff --git a/doc-src/_nav.html b/doc-src/_nav.html
index 0ae0fa67b..6c3afbe14 100644
--- a/doc-src/_nav.html
+++ b/doc-src/_nav.html
@@ -7,6 +7,7 @@
$!nav("mitmproxy.html", this, state)!$
$!nav("mitmdump.html", this, state)!$
+ $!nav("config.html", this, state)!$
$!nav("anticache.html", this, state)!$
diff --git a/doc-src/config.html b/doc-src/config.html
new file mode 100644
index 000000000..46688575d
--- /dev/null
+++ b/doc-src/config.html
@@ -0,0 +1,86 @@
+
+
+Mitmproxy is configured through a set of files in the users ~/.mitmproxy
+directory.
+
+
+
+
+ | mitmproxy.conf |
+ Settings for the mitmproxy. This file can contain any options supported by mitmproxy. |
+
+
+ | mitmdump.conf |
+ Settings for the mitmdump. This file can contain any options supported by mitmdump. |
+
+
+ | common.conf |
+
+ Settings shared between all command-line tools. Settings in
+ this file are over-ridden by those in the tool-specific
+ files. Only options shared by mitmproxy and mitmdump should be used in this file. |
+
+
+
+
+# Syntax
+
+## Comments
+
+
+# this is a comment
+; this is also a comment (.ini style)
+--- and this is a comment too (yaml style)
+
+
+## Key/Value pairs
+
+- Keys and values are case-sensitive
+- Whitespace is ignored
+- Lists are comma-delimited, and enclosed in square brackets
+
+
+name = value # (.ini style)
+name: value # (yaml style)
+--name value # (command-line option style)
+
+fruit = [apple, orange, lemon]
+indexes = [1, 12, 35 , 40]
+
+
+## Flags
+
+These are boolean options that take no value but true/false.
+
+
+name = true # (.ini style)
+name
+--name # (command-line option style)
+
+
+# Options
+
+The options available in the config files are precisely those available as
+command-line flags, with the key being the option's long name. To get a
+complete list of these, use the __--help__ option on each of the tools. Be
+careful to only specify common options in the __common.conf__ file -
+unsupported options in this file will be detected as an error on startup.
+
+# Examples
+
+## common.conf
+
+Note that __port__ is an option supported by all tools.
+
+
+port = 8080
+
+
+
+## mitmproxy.conf
+
+
+palette = light
+
+
+
diff --git a/doc-src/index.py b/doc-src/index.py
index 9eb0ea2bd..753f90a5c 100644
--- a/doc-src/index.py
+++ b/doc-src/index.py
@@ -4,10 +4,13 @@ import datetime
import countershape
from countershape import Page, Directory, markup, model
import countershape.template
-sys.path.insert(0, "..")
+
+MITMPROXY_SRC = os.path.abspath(
+ os.path.expanduser(os.environ.get("MITMPROXY_SRC", ".."))
+)
+sys.path.insert(0, MITMPROXY_SRC)
from libmproxy import filt, version
-MITMPROXY_SRC = os.environ.get("MITMPROXY_SRC", os.path.abspath(".."))
ns.VERSION = version.VERSION
if ns.options.website:
@@ -57,11 +60,13 @@ ns.navbar = countershape.template.File(None, "_nav.html")
pages = [
Page("index.html", "Introduction"),
Page("install.html", "Installation"),
- Page("mitmproxy.html", "mitmproxy"),
- Page("mitmdump.html", "mitmdump"),
Page("howmitmproxy.html", "How mitmproxy works"),
Page("modes.html", "Modes of Operation"),
+ Page("mitmproxy.html", "mitmproxy"),
+ Page("mitmdump.html", "mitmdump"),
+ Page("config.html", "configuration"),
+
Page("ssl.html", "Overview"),
Directory("certinstall"),
Directory("scripting"),
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py
index ec03d63ee..f5c66caab 100644
--- a/libmproxy/cmdline.py
+++ b/libmproxy/cmdline.py
@@ -194,6 +194,12 @@ def common_options(parser):
action= 'version',
version= "%(prog)s" + " " + version.VERSION
)
+ parser.add_argument(
+ '--shortversion',
+ action= 'version',
+ help = "show program's short version number and exit",
+ version = version.VERSION
+ )
parser.add_argument(
"--anticache",
action="store_true", dest="anticache", default=False,
@@ -451,9 +457,9 @@ def common_options(parser):
"--replay-ignore-payload-param",
action="append", dest="replay_ignore_payload_params", type=str,
help="""
- Request's payload parameters (application/x-www-form-urlencoded) to
- be ignored while searching for a saved flow to replay.
- Can be passed multiple times.
+ Request's payload parameters (application/x-www-form-urlencoded) to
+ be ignored while searching for a saved flow to replay.
+ Can be passed multiple times.
"""
)
diff --git a/libmproxy/onboarding/app.py b/libmproxy/onboarding/app.py
index 4023fae2e..f0aecc157 100644
--- a/libmproxy/onboarding/app.py
+++ b/libmproxy/onboarding/app.py
@@ -1,29 +1,80 @@
from __future__ import absolute_import
-import flask
import os
+import tornado.web
+import tornado.wsgi
+import tornado.template
+
+from .. import utils
from ..proxy import config
-mapp = flask.Flask(__name__)
-mapp.debug = True
+
+loader = tornado.template.Loader(utils.pkg_data.path("onboarding/templates"))
-def master():
- return flask.request.environ["mitmproxy.master"]
+class Adapter(tornado.wsgi.WSGIAdapter):
+ # Tornado doesn't make the WSGI environment available to pages, so this
+ # hideous monkey patch is the easiest way to get to the mitmproxy.master
+ # variable.
+ def __init__(self, application):
+ self._application = application
+
+ def application(self, request):
+ request.master = self.environ["mitmproxy.master"]
+ return self._application(request)
+
+ def __call__(self, environ, start_response):
+ self.environ = environ
+ return tornado.wsgi.WSGIAdapter.__call__(
+ self,
+ environ,
+ start_response
+ )
-@mapp.route("/")
-def index():
- return flask.render_template("index.html", section="home")
+class Index(tornado.web.RequestHandler):
+ def get(self):
+ t = loader.load("index.html")
+ self.write(t.generate())
-@mapp.route("/cert/pem")
-def certs_pem():
- p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.pem")
- return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert')
+class PEM(tornado.web.RequestHandler):
+ def get(self):
+ p = os.path.join(
+ self.request.master.server.config.cadir,
+ config.CONF_BASENAME + "-ca-cert.pem"
+ )
+ self.set_header(
+ "Content-Type", "application/x-x509-ca-cert"
+ )
+ self.write(open(p, "rb").read())
-@mapp.route("/cert/p12")
-def certs_p12():
- p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.p12")
- return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12')
+class P12(tornado.web.RequestHandler):
+ def get(self):
+ p = os.path.join(
+ self.request.master.server.config.cadir,
+ config.CONF_BASENAME + "-ca-cert.p12"
+ )
+ self.set_header(
+ "Content-Type", "application/x-pkcs12"
+ )
+ self.write(open(p, "rb").read())
+
+
+application = tornado.web.Application(
+ [
+ (r"/", Index),
+ (r"/cert/pem", PEM),
+ (r"/cert/p12", P12),
+ (
+ r"/static/(.*)",
+ tornado.web.StaticFileHandler,
+ {
+ "path": utils.pkg_data.path("onboarding/static")
+ }
+ ),
+ ],
+ #debug=True
+)
+mapp = Adapter(application)
diff --git a/libmproxy/onboarding/templates/frame.html b/libmproxy/onboarding/templates/frame.html
index b5c5c67c8..f00e1a66e 100644
--- a/libmproxy/onboarding/templates/frame.html
+++ b/libmproxy/onboarding/templates/frame.html
@@ -3,7 +3,7 @@
{% block body %}
- {% endblock %}
+ {% end %}
-{% endblock %}
+{% end %}
diff --git a/libmproxy/onboarding/templates/index.html b/libmproxy/onboarding/templates/index.html
index 65fda5d2b..1bcff1b88 100644
--- a/libmproxy/onboarding/templates/index.html
+++ b/libmproxy/onboarding/templates/index.html
@@ -32,4 +32,4 @@
between mitmproxy installations.
-{% endblock %}
+{% end %}
diff --git a/libmproxy/onboarding/templates/layout.html b/libmproxy/onboarding/templates/layout.html
index 622858bea..8726a788e 100644
--- a/libmproxy/onboarding/templates/layout.html
+++ b/libmproxy/onboarding/templates/layout.html
@@ -25,7 +25,7 @@
{% block content %}
- {% endblock %}
+ {% end %}
diff --git a/release/mitmdump.spec b/release/mitmdump.spec
new file mode 100644
index 000000000..a84908f56
--- /dev/null
+++ b/release/mitmdump.spec
@@ -0,0 +1,32 @@
+# -*- mode: python -*-
+
+from glob import glob
+block_cipher = None
+
+a = Analysis(['./mitmdump'],
+ hiddenimports=[],
+ hookspath=None,
+ runtime_hooks=None,
+ excludes=None,
+ cipher=block_cipher,
+ )
+a.datas = Tree(
+ "./libmproxy/onboarding/templates",
+ prefix="libmproxy/onboarding/templates"
+)
+a.datas += Tree(
+ "./libmproxy/onboarding/static",
+ prefix="libmproxy/onboarding/static"
+)
+pyz = PYZ(a.pure,
+ cipher=block_cipher)
+exe = EXE(pyz,
+ a.scripts,
+ a.binaries,
+ a.zipfiles,
+ a.datas,
+ name='mitmdump',
+ debug=False,
+ strip=None,
+ upx=True,
+ console=True )
diff --git a/release/mitmproxy.spec b/release/mitmproxy.spec
new file mode 100644
index 000000000..546c28991
--- /dev/null
+++ b/release/mitmproxy.spec
@@ -0,0 +1,32 @@
+# -*- mode: python -*-
+
+from glob import glob
+block_cipher = None
+
+a = Analysis(['./mitmproxy'],
+ hiddenimports=[],
+ hookspath=None,
+ runtime_hooks=None,
+ excludes=None,
+ cipher=block_cipher,
+ )
+a.datas = Tree(
+ "./libmproxy/onboarding/templates",
+ prefix="libmproxy/onboarding/templates"
+)
+a.datas += Tree(
+ "./libmproxy/onboarding/static",
+ prefix="libmproxy/onboarding/static"
+)
+pyz = PYZ(a.pure,
+ cipher=block_cipher)
+exe = EXE(pyz,
+ a.scripts,
+ a.binaries,
+ a.zipfiles,
+ a.datas,
+ name='mitmproxy',
+ debug=False,
+ strip=None,
+ upx=True,
+ console=True )
diff --git a/release/mitmweb.spec b/release/mitmweb.spec
new file mode 100644
index 000000000..4e99da38a
--- /dev/null
+++ b/release/mitmweb.spec
@@ -0,0 +1,40 @@
+# -*- mode: python -*-
+
+from glob import glob
+block_cipher = None
+
+a = Analysis(['./mitmweb'],
+ hiddenimports=[],
+ hookspath=None,
+ runtime_hooks=None,
+ excludes=None,
+ cipher=block_cipher,
+ )
+a.datas = Tree(
+ "./libmproxy/onboarding/templates",
+ prefix="libmproxy/onboarding/templates"
+)
+a.datas += Tree(
+ "./libmproxy/onboarding/static",
+ prefix="libmproxy/onboarding/static"
+)
+a.datas += Tree(
+ "./libmproxy/web/templates",
+ prefix="libmproxy/web/templates"
+)
+a.datas += Tree(
+ "./libmproxy/web/static",
+ prefix="libmproxy/web/static"
+)
+pyz = PYZ(a.pure,
+ cipher=block_cipher)
+exe = EXE(pyz,
+ a.scripts,
+ a.binaries,
+ a.zipfiles,
+ a.datas,
+ name='mitmweb',
+ debug=False,
+ strip=None,
+ upx=True,
+ console=True )
diff --git a/release/osx-binaries b/release/osx-binaries
index 9945e471b..862ddd59b 100755
--- a/release/osx-binaries
+++ b/release/osx-binaries
@@ -5,27 +5,45 @@
# A few quirks to note, which should be re-checked every release:
# - We require the latest development version of PyInstaller.
-# - PyInstaller has trouble detecting the zope.interfaces package. This is
-# required by Twisted, which for mysterious reasons is required by Urwid. The
-# answer is to touch the __init__.py file in the zope directory. On my system:
-# touch /Library/Python/2.7/site-packages/zope/__init__.py
+# To run, first install netlib and mitmproxy, then run
+#
+# ./release/osx-binaries
+#
+# From the top-level mitmproxy directory.
-# To run, first install netlib and mitmproxy, then change into the pyinstaller
-# directory, and then run this script.
+usage ()
+{
+ echo 'Usage : ./release/osx-binaries /path/to/pyinstaller.py'
+ echo 'Run from the top-level mitmproxy directory'
+ exit
+}
-DST=/tmp/osx-mitmproxy
-MITMPROXY=~/mitmproxy/mitmproxy
-PYINST_CMD="./pyinstaller.py -F --clean"
+if [ "$1" = "" ]
+then
+ usage
+fi
-rm -rf $DST
-mkdir -p $DST
-rm -rf mitmproxy
-rm -rf mitmdump
-$PYINST_CMD $MITMPROXY/mitmproxy
-cp mitmproxy/dist/mitmproxy $DST
+TMPDIR=./tmp
+PYINST_CMD=$1" -F --clean"
-$PYINST_CMD $MITMPROXY/mitmdump
-cp mitmdump/dist/mitmdump $DST
+rm -f dist/*
+rm -rf $TMPDIR
-cshape $MITMPROXY/doc-src $DST/doc
+$PYINST_CMD ./release/mitmdump.spec
+./dist/mitmdump --version || exit 1
+
+$PYINST_CMD ./release/mitmproxy.spec
+./dist/mitmproxy --version || exit 1
+
+$PYINST_CMD ./release/mitmweb.spec
+./dist/mitmweb --version || exit 1
+
+DST=osx-mitmproxy-`./dist/mitmdump --shortversion 2>&1`
+mkdir -p $TMPDIR/$DST
+cp ./dist/mitmproxy $TMPDIR/$DST
+cp ./dist/mitmdump $TMPDIR/$DST
+cshape ./doc-src $TMPDIR/$DST/doc
+
+cd $TMPDIR
+tar -czvf $DST.tar.gz $DST
diff --git a/release/release-checklist b/release/release-checklist
index 31a1a48fd..154103e9c 100644
--- a/release/release-checklist
+++ b/release/release-checklist
@@ -1,5 +1,5 @@
-- Bump the version number:
+- Check the version number:
mitmproxy/libmproxy/version.py
netlib/netlib/version.py
@@ -11,30 +11,24 @@
- Run the test release, make sure the output is sensible
./release/test-release
-- Build sdist packages:
- python ./setup.py sdist
-
-- Test the packages by installing in a virtualenv:
- - cd dist
- - tar -xzvf pkgfile.tgz
- - virtualenv venv
-
- Build the OSX binaries
- Follow instructions in osxbinaries
- - Package:
- cp -r ./doc /tmp/osx-mitmproxy/
- mv /tmp/osx-mitmproxy /tmp/osx-mitmproxy-VERSION
- tar -czvf /tmp/osx-mitmproxy-VERSION.tar.gz /tmp/osx-mitmproxy-VERSION
- mv /tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download
+ - Move to download dir:
+ mv ./tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download
- Build the sources for each project:
python ./setup.py sdist
mv ./dist/FILE ~/mitmproxy/www.mitmproxy.org/src/download
-
-- Adjust links on www.mitmproxy.org
+- Tag with the version number, and do:
+ git push --tags
- Upload to pypi for each project:
python ./setup.py sdist upload
+- Now bump the version number to be ready for the next cycle:
+
+ mitmproxy/libmproxy/version.py
+ netlib/netlib/version.py
+ pathod/libpathod/version.py
diff --git a/setup.py b/setup.py
index 79398a186..953df2bb2 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,6 @@ deps = {
"netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION),
"pyasn1>0.1.2",
"pyOpenSSL>=0.14",
- "Flask>=0.10.1",
"tornado>=4.0.2",
"configargparse>=0.9.3"
}
diff --git a/test/tservers.py b/test/tservers.py
index 12154ba7d..37929d1ab 100644
--- a/test/tservers.py
+++ b/test/tservers.py
@@ -158,10 +158,10 @@ class HTTPProxTest(ProxTestBase):
if self.ssl:
p = libpathod.pathoc.Pathoc(("127.0.0.1", self.proxy.port), True)
p.connect((APP_HOST, APP_PORT))
- return p.request("get:'/%s'"%page)
+ return p.request("get:'%s'"%page)
else:
p = self.pathoc()
- return p.request("get:'http://%s/%s'"%(APP_HOST, page))
+ return p.request("get:'http://%s%s'"%(APP_HOST, page))
class TResolver: