Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2025-01-30 23:09:44 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1602 from cortesi/scriptdocs

Browse Source 
docs: stub out new script documentation
This commit is contained in:
Aldo Cortesi 2016-10-16 20:50:09 +13:00 committed by GitHub
commit e4a32d6844
30 changed files with 626 additions and 468 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
docs/_static/theme_overrides.css vendored
View File

@ -4,6 +4,10 @@
white-space: normal;
}
.wy-table-responsive > table > tbody > tr > td {
vertical-align: top !important;
}
.wy-table-responsive {
margin-bottom: 24px;
max-width: 100%;
@ -13,3 +17,28 @@
.wy-menu-vertical header, .wy-menu-vertical p.caption {
color: #e0e0e0;
}
.code-block-caption {
height: 1.5em;
}
.code-block-caption .caption-text {
font-size: 0.8em;
float: right;
}
.code-block-caption .headerlink {
display: none !important;
}
.function .headerlink {
display: none !important;
}
dl .reference.internal {
display: none !important;
}
dl .headerlink {
display: none !important;
}

57
docs/conf.py
View File

@ -1,11 +1,18 @@
import sys
import importlib
import inspect
import os
import subprocess
import sys
sys.path.insert(0, os.path.abspath('..'))
import netlib.version
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.doctest',
'sphinx.ext.extlinks',
'sphinx.ext.linkcode',
'sphinx.ext.viewcode',
'sphinx.ext.napoleon',
'sphinxcontrib.documentedlist'
@ -156,7 +163,7 @@ html_static_path = ['_static']
#html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
html_show_sourcelink = False
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
@ -189,5 +196,51 @@ html_static_path = ['_static']
# Output file base name for HTML help builder.
htmlhelp_basename = 'mitmproxydoc'
last_tag, tag_dist, commit = (
subprocess.check_output(["git", "describe", "--tags", "--long"])
.decode()
.strip()
.rsplit("-", 2)
)
tag_dist = int(tag_dist)
if tag_dist == 0:
tag = last_tag
else:
tag = "master"
SRCBASE = "https://github.com/mitmproxy/mitmproxy/blob/{}".format(tag)
extlinks = dict(
src = (SRCBASE + r"/%s", '')
)
def linkcode_resolve(domain, info):
if domain != 'py':
return None
module, fullname = info['module'], info['fullname']
if not module:
return None
obj = importlib.import_module(module)
for item in fullname.split('.'):
obj = getattr(obj, item, None)
if obj is None:
return None
try:
spath = inspect.getsourcefile(obj)
_, line = inspect.getsourcelines(obj)
except (TypeError, IOError):
return None
if spath.rfind("netlib") > -1:
off = spath.rfind("netlib")
mpath = spath[off:]
elif spath.rfind("mitmproxy") > -1:
off = spath.rfind("mitmproxy")
mpath = spath[off:]
else:
return None
return SRCBASE + "/%s#L%s" % (mpath, line)
def setup(app):
app.add_stylesheet('theme_overrides.css')

81
docs/dev/models.rst
View File

@ -1,81 +0,0 @@
.. _models:
Datastructures
==============
.. automodule:: mitmproxy.models
:members: HTTPFlow, HTTPRequest, HTTPResponse
.. automodule:: netlib.http
.. autoclass:: Request
.. rubric:: Data
.. autoattribute:: first_line_format
.. autoattribute:: method
.. autoattribute:: scheme
.. autoattribute:: host
.. autoattribute:: port
.. autoattribute:: path
.. autoattribute:: http_version
.. autoattribute:: headers
.. autoattribute:: content
.. autoattribute:: timestamp_start
.. autoattribute:: timestamp_end
.. rubric:: Computed Properties and Convenience Methods
.. autoattribute:: text
.. autoattribute:: url
.. autoattribute:: pretty_host
.. autoattribute:: pretty_url
.. autoattribute:: query
.. autoattribute:: cookies
.. autoattribute:: path_components
.. automethod:: anticache
.. automethod:: anticomp
.. automethod:: constrain_encoding
.. autoattribute:: urlencoded_form
.. autoattribute:: multipart_form
.. autoclass:: Response
.. automethod:: make
.. rubric:: Data
.. autoattribute:: http_version
.. autoattribute:: status_code
.. autoattribute:: reason
.. autoattribute:: headers
.. autoattribute:: content
.. autoattribute:: timestamp_start
.. autoattribute:: timestamp_end
.. rubric:: Computed Properties and Convenience Methods
.. autoattribute:: text
.. autoattribute:: cookies
.. autoclass:: Headers
:members:
:special-members:
:no-undoc-members:
.. automodule:: netlib.multidict
.. autoclass:: MultiDictView
.. automethod:: get_all
.. automethod:: set_all
.. automethod:: add
.. automethod:: insert
.. automethod:: keys
.. automethod:: values
.. automethod:: items
.. automethod:: to_dict
.. autoclass:: mitmproxy.models.Error
:show-inheritance:
.. autoclass:: mitmproxy.models.ServerConnection
:show-inheritance:
.. autoclass:: mitmproxy.models.ClientConnection
:show-inheritance:

2
docs/features/clientreplay.rst
View File

@ -14,5 +14,5 @@ You may want to use client-side replay in conjunction with the
================== ===========
command-line ``-c path``
mitmproxy shortcut :kbd:`c`
mitmproxy shortcut :kbd:`R` then :kbd:`c`
================== ===========

6
docs/features/responsestreaming.rst
View File

@ -35,10 +35,10 @@ command-line ``--stream SIZE``
Customizing Response Streaming
------------------------------
You can also use an :ref:`inlinescripts` to customize exactly
which responses are streamed.
You can also use a script to customize exactly which responses are streamed.
Responses that should be tagged for streaming by setting their ``.stream`` attribute to ``True``:
Responses that should be tagged for streaming by setting their ``.stream``
attribute to ``True``:
.. literalinclude:: ../../examples/stream.py
:caption: examples/stream.py

2
docs/features/serverreplay.rst
View File

@ -35,5 +35,5 @@ the :kbd:`o` options shortcut within :program:`mitmproxy`.
================== ===========
command-line ``-S path``
mitmproxy shortcut :kbd:`S`
mitmproxy shortcut :kbd:`R` then :kbd:`s`
================== ===========

6
docs/index.rst
View File

@ -51,9 +51,9 @@
:hidden:
:caption: Scripting
scripting/inlinescripts
dev/models
scripting/mitmproxy
scripting/overview
scripting/events
scripting/api
.. toctree::

40
docs/scripting/api.rst Normal file
View File

@ -0,0 +1,40 @@
.. _api:
API
===
- Errors
- `mitmproxy.models.flow.Error <#mitmproxy.models.flow.Error>`_
- HTTP
- `mitmproxy.models.http.HTTPRequest <#mitmproxy.models.http.HTTPRequest>`_
- `mitmproxy.models.http.HTTPResponse <#mitmproxy.models.http.HTTPResponse>`_
- `mitmproxy.models.http.HTTPFlow <#mitmproxy.models.http.HTTPFlow>`_
- Logging
- `mitmproxy.controller.Log <#mitmproxy.controller.Log>`_
- `mitmproxy.controller.LogEntry <#mitmproxy.controller.LogEntry>`_
Errors
------
.. autoclass:: mitmproxy.models.flow.Error
:inherited-members:
HTTP
----
.. autoclass:: mitmproxy.models.http.HTTPRequest
:inherited-members:
.. autoclass:: mitmproxy.models.http.HTTPResponse
:inherited-members:
.. autoclass:: mitmproxy.models.http.HTTPFlow
:inherited-members:
Logging
--------
.. autoclass:: mitmproxy.controller.Log
:inherited-members:

202
docs/scripting/events.rst Normal file
View File

@ -0,0 +1,202 @@
.. _events:
Events
=======
General
-------
.. list-table::
:widths: 40 60
:header-rows: 0
* - .. py:function:: configure(options, updated)
- Called once on startup, and whenever options change.
*options*
An ``options.Options`` object with the total current configuration
state of mitmproxy.
*updated*
A set of strings indicating which configuration options have been
updated. This contains all options when *configure* is called on
startup, and only changed options subsequently.
* - .. py:function:: done()
- Called once when the script shuts down, either because it's been
unloaded, or because the proxy itself is shutting down.
* - .. py:function:: log(entry)
- Called whenever an event log is added.
*entry*
An ``controller.LogEntry`` object - ``entry.msg`` is the log text,
and ``entry.level`` is the urgency level ("debug", "info", "warn",
"error").
* - .. py:function:: start()
- Called once on startup, before any other events. If you return a
value from this event, it will replace the current addon. This
allows you to, "boot into" an addon implemented as a class instance
from the module level.
* - .. py:function:: tick()
- Called at a regular sub-second interval as long as the addon is
executing.
Connection
----------
.. list-table::
:widths: 40 60
:header-rows: 0
* - .. py:function:: clientconnect(root_layer)
- Called when a client initiates a connection to the proxy. Note that a
connection can correspond to multiple HTTP requests.
*root_layer*
The root layer (see `mitmproxy.protocol` for an explanation what
the root layer is), provides transparent access to all attributes
of the :py:class:`~mitmproxy.proxy.RootContext`. For example,
``root_layer.client_conn.address`` gives the remote address of the
connecting client.
* - .. py:function:: clientdisconnect(root_layer)
- Called when a client disconnects from the proxy.
*root_layer*
The root layer object.
* - .. py:function:: next_layer(layer)
- Called whenever layers are switched. You may change which layer will
be used by returning a new layer object from this event.
*layer*
The next layer, as determined by mitmpmroxy.
* - .. py:function:: serverconnect(server_conn)
- Called before the proxy initiates a connection to the target server.
Note that a connection can correspond to multiple HTTP requests.
*server_conn*
A ``ServerConnection`` object. It is guaranteed to have a non-None
``address`` attribute.
* - .. py:function:: serverdisconnect(server_conn)
- Called when the proxy has closed the server connection.
*server_conn*
A ``ServerConnection`` object.
HTTP Events
-----------
.. list-table::
:widths: 40 60
:header-rows: 0
* - .. py:function:: request(flow)
- Called when a client request has been received.
*flow*
A ``models.HTTPFlow`` object. At this point, the flow is
guaranteed to have a non-None ``request`` attribute.
* - .. py:function:: requestheaders(flow)
- Called when the headers of a client request have been received, but
before the request body is read.
*flow*
A ``models.HTTPFlow`` object. At this point, the flow is
guaranteed to have a non-None ``request`` attribute.
* - .. py:function:: responseheaders(flow)
- Called when the headers of a server response have been received, but
before the response body is read.
*flow*
A ``models.HTTPFlow`` object. At this point, the flow is
guaranteed to have a non-none ``request`` and ``response``
attributes, however the response will have no content.
* - .. py:function:: response(flow)
- Called when a server response has been received.
*flow*
A ``models.HTTPFlow`` object. At this point, the flow is
guaranteed to have a non-none ``request`` and ``response``
attributes. The raw response body will be in ``response.body``,
unless response streaming has been enabled.
* - .. py:function:: error(flow)
- Called when a flow error has occurred, e.g. invalid server responses,
or interrupted connections. This is distinct from a valid server HTTP
error response, which is simply a response with an HTTP error code.
*flow*
The flow containing the error. It is guaranteed to have
non-None ``error`` attribute.
WebSocket Events
-----------------
.. list-table::
:widths: 40 60
:header-rows: 0
* - .. py:function:: websockets_handshake(flow)
- Called when a client wants to establish a WebSockets connection. The
WebSockets-specific headers can be manipulated to manipulate the
handshake. The ``flow`` object is guaranteed to have a non-None
``request`` attribute.
*flow*
The flow containing the HTTP websocket handshake request. The
object is guaranteed to have a non-None ``request`` attribute.
TCP Events
----------
These events are called only if the connection is in :ref:`TCP mode
<tcpproxy>`. So, for instance, TCP events are not called for ordinary HTTP/S
connections.
.. list-table::
:widths: 40 60
:header-rows: 0
* - .. py:function:: tcp_end(flow)
- Called when TCP streaming ends.
*flow*
A ``models.TCPFlow`` object.
* - .. py:function:: tcp_error(flow)
- Called when a TCP error occurs - e.g. the connection closing
unexpectedly.
*flow*
A ``models.TCPFlow`` object.
* - .. py:function:: tcp_message(flow)
- Called a TCP payload is received from the client or server. The
sender and receiver are identifiable. The most recent message will be
``flow.messages[-1]``. The message is user-modifiable.
*flow*
A ``models.TCPFlow`` object.
* - .. py:function:: tcp_start(flow)
- Called when TCP streaming starts.
*flow*
A ``models.TCPFlow`` object.

227
docs/scripting/inlinescripts.rst
View File

@ -1,227 +0,0 @@
.. _inlinescripts:
Inline Scripts
==============
**mitmproxy** has a powerful scripting API that allows you to modify flows
on-the-fly or rewrite previously saved flows locally.
The mitmproxy scripting API is event driven - a script is simply a Python
module that exposes a set of event methods. Here's a complete mitmproxy script
that adds a new header to every HTTP response before it is returned to the
client:
.. literalinclude:: ../../examples/add_header.py
:caption: examples/add_header.py
:language: python
All events that deal with an HTTP request get an instance of :py:class:`~mitmproxy.models.HTTPFlow`,
which we can use to manipulate the response itself.
We can now run this script using mitmdump or mitmproxy as follows:
>>> mitmdump -s add_header.py
The new header will be added to all responses passing through the proxy.
Examples
--------
mitmproxy comes with a variety of example inline scripts, which demonstrate many basic tasks.
We encourage you to either browse them locally or on `GitHub`_.
Events
------
Script Lifecycle Events
^^^^^^^^^^^^^^^^^^^^^^^
.. py:function:: start(context)
Called once on startup, before any other events.
:param List[str] argv: The inline scripts' arguments.
For example, ``mitmproxy -s 'example.py --foo 42'`` sets argv to ``["--foo", "42"]``.
.. py:function:: done(context)
Called once on script shutdown, after any other events.
Connection Events
^^^^^^^^^^^^^^^^^
.. py:function:: clientconnect(context, root_layer)
Called when a client initiates a connection to the proxy. Note that
a connection can correspond to multiple HTTP requests.
.. versionchanged:: 0.14
:param Layer root_layer: The root layer, which provides transparent access to all attributes of the
:py:class:`~mitmproxy.proxy.RootContext`. For example, ``root_layer.client_conn.address``
gives the remote address of the connecting client.
.. py:function:: clientdisconnect(context, root_layer)
Called when a client disconnects from the proxy.
.. versionchanged:: 0.14
:param Layer root_layer: see :py:func:`clientconnect`
.. py:function:: serverconnect(context, server_conn)
Called before the proxy initiates a connection to the target server. Note that
a connection can correspond to multiple HTTP requests.
:param ServerConnection server_conn: The server connection object. It is guaranteed to have a
non-None ``address`` attribute.
.. py:function:: serverdisconnect(context, server_conn)
Called when the proxy has closed the server connection.
.. versionadded:: 0.14
:param ServerConnection server_conn: see :py:func:`serverconnect`
HTTP Events
^^^^^^^^^^^
.. py:function:: request(context, flow)
Called when a client request has been received. The ``flow`` object is
guaranteed to have a non-None ``request`` attribute.
:param HTTPFlow flow: The flow containing the request which has been received.
The object is guaranteed to have a non-None ``request`` attribute.
.. py:function:: responseheaders(context, flow)
Called when the headers of a server response have been received.
This will always be called before the response hook.
:param HTTPFlow flow: The flow containing the request and response.
The object is guaranteed to have non-None ``request`` and
``response`` attributes. ``response.content`` will be ``None``,
as the response body has not been read yet.
.. py:function:: response(context, flow)
Called when a server response has been received.
:param HTTPFlow flow: The flow containing the request and response.
The object is guaranteed to have non-None ``request`` and
``response`` attributes. ``response.body`` will contain the raw response body,
unless response streaming has been enabled.
.. py:function:: error(context, flow)
Called when a flow error has occurred, e.g. invalid server responses, or
interrupted connections. This is distinct from a valid server HTTP error
response, which is simply a response with an HTTP error code.
:param HTTPFlow flow: The flow containing the error.
It is guaranteed to have non-None ``error`` attribute.
WebSockets Events
^^^^^^^^^^^^^^^^^
.. py:function:: websocket_handshake(context, flow)
Called when a client wants to establish a WebSockets connection.
The WebSockets-specific headers can be manipulated to manipulate the handshake.
The ``flow`` object is guaranteed to have a non-None ``request`` attribute.
:param HTTPFlow flow: The flow containing the request which has been received.
The object is guaranteed to have a non-None ``request`` attribute.
TCP Events
^^^^^^^^^^
.. py:function:: tcp_message(context, tcp_msg)
.. warning:: API is subject to change
If the proxy is in :ref:`TCP mode <tcpproxy>`, this event is called when it
receives a TCP payload from the client or server.
The sender and receiver are identifiable. The message is user-modifiable.
:param TcpMessage tcp_msg: see *examples/tcp_message.py*
API
---
The canonical API documentation is the code, which you can browse here, locally or on `GitHub`_.
*Use the Source, Luke!*
The main classes you will deal with in writing mitmproxy scripts are:
:py:class:`mitmproxy.flow.FlowMaster`
- The "heart" of mitmproxy, usually subclassed as :py:class:`mitmproxy.dump.DumpMaster` or
:py:class:`mitmproxy.console.ConsoleMaster`.
:py:class:`~mitmproxy.models.ClientConnection`
- Describes a client connection.
:py:class:`~mitmproxy.models.ServerConnection`
- Describes a server connection.
:py:class:`~mitmproxy.models.HTTPFlow`
- A collection of objects representing a single HTTP transaction.
:py:class:`~mitmproxy.models.HTTPRequest`
- An HTTP request.
:py:class:`~mitmproxy.models.HTTPResponse`
- An HTTP response.
:py:class:`~mitmproxy.models.Error`
- A communications error.
:py:class:`netlib.http.Headers`
- A dictionary-like object for managing HTTP headers.
:py:class:`netlib.certutils.SSLCert`
- Exposes information SSL certificates.
Running scripts in parallel
---------------------------
We have a single flow primitive, so when a script is blocking, other requests are not processed.
While that's usually a very desirable behaviour, blocking scripts can be run threaded by using the
:py:obj:`mitmproxy.script.concurrent` decorator.
**If your script does not block, you should avoid the overhead of the decorator.**
.. literalinclude:: ../../examples/nonblocking.py
:caption: examples/nonblocking.py
:language: python
Make scripts configurable with arguments
----------------------------------------
Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by
surrounding the script call with quotes, e.g. ```mitmdump -s 'script.py --foo 42'``.
The arguments are then exposed in the start event:
.. literalinclude:: ../../examples/modify_response_body.py
:caption: examples/modify_response_body.py
:language: python
Running scripts on saved flows
------------------------------
Sometimes, we want to run a script on :py:class:`~mitmproxy.models.Flow` objects that are already
complete. This happens when you start a script, and then load a saved set of flows from a file
(see the "scripted data transformation" example :ref:`here <mitmdump>`).
It also happens when you run a one-shot script on a single flow through the ``|`` (pipe) shortcut
in mitmproxy.
In this case, there are no client connections, and the events are run in the following order:
**start**, **request**, **responseheaders**, **response**, **error**, **done**.
If the flow doesn't have a **response** or **error** associated with it, the matching events will
be skipped.
Spaces in the script path
-------------------------
By default, spaces are interpreted as a separator between the inline script and its arguments
(e.g. ``-s 'foo.py 42'``). Consequently, the script path needs to be wrapped in a separate pair of
quotes if it contains spaces: ``-s '\'./foo bar/baz.py\' 42'``.
.. _GitHub: https://github.com/mitmproxy/mitmproxy

26
docs/scripting/mitmproxy.rst
View File

@ -1,26 +0,0 @@
FlowMaster
==========
.. note::
We strongly encourage you to use :ref:`inlinescripts` rather than subclassing mitmproxy's FlowMaster.
- Inline Scripts are equally powerful and provide an easier syntax.
- Most examples are written as inline scripts.
- Multiple inline scripts can be used together.
- Inline Scripts can either be executed headless with mitmdump or within the mitmproxy UI.
All of mitmproxy's basic functionality is exposed through the **mitmproxy**
library. The example below shows a simple implementation of the "sticky cookie"
functionality included in the interactive mitmproxy program. Traffic is
monitored for ``Cookie`` and ``Set-Cookie`` headers, and requests are rewritten
to include a previously seen cookie if they don't already have one. In effect,
this lets you log in to a site using your browser, and then make subsequent
requests using a tool like curl, which will then seem to be part of the
authenticated session.
.. literalinclude:: ../../examples/stickycookies
:caption: examples/stickycookies
:language: python

141
docs/scripting/overview.rst Normal file
View File

@ -0,0 +1,141 @@
.. _overview:
Overview
========
Mitmproxy has a powerful scripting API that allows you to control almost any
aspect of traffic being proxied. In fact, much of mitmproxy's own core
functionality is implemented using the exact same API exposed to scripters (see
:src:`mitmproxy/builtins`).
A simple example
----------------
Scripting is event driven, with named handlers on the script object called at
appropriate points of mitmproxy's operation. Here's a complete mitmproxy script
that adds a new header to every HTTP response before it is returned to the
client:
.. literalinclude:: ../../examples/add_header.py
:caption: :src:`examples/add_header.py`
:language: python
All events that deal with an HTTP request get an instance of `HTTPFlow
<api.html#mitmproxy.models.http.HTTPFlow>`_, which we can use to manipulate the
response itself. We can now run this script using mitmdump, and the new header
will be added to all responses passing through the proxy:
>>> mitmdump -s add_header.py
Using classes
-------------
In the example above, the script object is the ``add_header`` module itself.
That is, the handlers are declared at the global level of the script. This is
great for quick hacks, but soon becomes limiting as scripts become more
sophisticated.
When a script first starts up, the `start <events.html#start>`_, event is
called before anything else happens. You can replace the current script object
by returning it from this handler. Here's how this looks when applied to the
example above:
.. literalinclude:: ../../examples/classes.py
:caption: :src:`examples/classes.py`
:language: python
So here, we're using a module-level script to "boot up" into a class instance.
From this point on, the module-level script is removed from the handler chain,
and is replaced by the class instance.
Handling arguments
------------------
Scripts can handle their own command-line arguments, just like any other Python
program. Let's build on the example above to do something slightly more
sophisticated - replace one value with another in all responses. Mitmproxy's
`HTTPRequest <api.html#mitmproxy.models.http.HTTPRequest>`_ and `HTTPResponse
<api.html#mitmproxy.models.http.HTTPResponse>`_ objects have a handy `replace
<api.html#mitmproxy.models.http.HTTPResponse.replace>`_ method that takes care
of all the details for us.
.. literalinclude:: ../../examples/arguments.py
:caption: :src:`examples/arguments.py`
:language: python
We can now call this script on the command-line like this:
>>> mitmdump -dd -s "./arguments.py html faketml"
Whenever a handler is called, mitpmroxy rewrites the script environment so that
it sees its own arguments as if it was invoked from the command-line.
Logging and the context
-----------------------
Scripts should not output straight to stderr or stdout. Instead, the `log
<api.html#mitmproxy.controller.Log>`_ object on the ``ctx`` contexzt module
should be used, so that the mitmproxy host program can handle output
appropriately. So, mitmdump can print colorised sript output to the terminal,
and mitmproxy console can place script output in the event buffer.
Here's how this looks:
.. literalinclude:: ../../examples/logging.py
:caption: :src:`examples/logging.py`
:language: python
The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master``
for advanced usage.
Running scripts on saved flows
------------------------------
When a flow is loaded from disk, the sequence of events that the flow would
have gone through on the wire is partially replayed. So, for instance, an HTTP
flow loaded from disk will trigger `requestheaders
<events.html#requestheaders>`_, `request <events.html#request>`_,
`responseheaders <events.html#responseheaders>`_ and `response
<events.html#response>`_ in order. We can use this behaviour to transform saved
traffic using scripts. For example, we can invoke the replacer script from
above on saved traffic as follows:
>>> mitmdump -dd -s "./arguments.py html fakehtml" -r saved -w changed
This command starts the ``arguments`` script, reads all the flows from
``saved`` transforming them in the process, then writes them all to
``changed``.
The mitmproxy console tool provides interactive ways to run transforming
scripts on flows - for instance, you can run a one-shot script on a single flow
through the ``|`` (pipe) shortcut.
Concurrency
-----------
The mitmproxy script mechanism is single threaded, and the proxy blocks while
script handlers execute. This hugely simplifies the most common case, where
handlers are light-weight and the blocking doesn't have a performance impact.
It's possible to implement a concurrent mechanism on top of the blocking
framework, and mitmproxy includes a handy example of this that is fit for most
purposes. You can use it as follows:
.. literalinclude:: ../../examples/nonblocking.py
:caption: :src:`examples/nonblocking.py`
:language: python
Developing scripts
------------------
Mitmprxoy monitors scripts for modifications, and reloads them on change. When
this happens, the script is shut down (the `done <events.html#done>`_ event is
called), and the new instance is started up as if the script had just been
loaded (the `start <events.html#start>`_ and `configure
<events.html#configure>`_ events are called).

17
examples/arguments.py Normal file
View File

@ -0,0 +1,17 @@
import argparse
class Replacer:
def __init__(self, src, dst):
self.src, self.dst = src, dst
def response(self, flow):
flow.response.replace(self.src, self.dst)
def start():
parser = argparse.ArgumentParser()
parser.add_argument("src", type=str)
parser.add_argument("dst", type=str)
args = parser.parse_args()
return Replacer(args.src, args.dst)

7
examples/classes.py Normal file
View File

@ -0,0 +1,7 @@
class AddHeader:
def response(self, flow):
flow.response.headers["newheader"] = "foo"
def start():
return AddHeader()

6
examples/logging.py Normal file
View File

@ -0,0 +1,6 @@
from mitmproxy import ctx
def start():
ctx.log.info("This is some informative text.")
ctx.log.error("This is an error.")

21
examples/modify_response_body.py
View File

@ -1,21 +0,0 @@
# Usage: mitmdump -s "modify_response_body.py mitmproxy bananas"
# (this script works best with --anticache)
import sys
state = {}
def start():
if len(sys.argv) != 3:
raise ValueError('Usage: -s "modify_response_body.py old new"')
# You may want to use Python's argparse for more sophisticated argument
# parsing.
state["old"], state["new"] = sys.argv[1].encode(), sys.argv[2].encode()
def response(flow):
flow.response.content = flow.response.content.replace(
state["old"],
state["new"]
)

26
mitmproxy/addons.py
View File

@ -28,20 +28,32 @@ class Addons(object):
with self.master.handlecontext():
i.configure(options, updated)
def startup(self, s):
"""
Run startup events on addon.
"""
self.invoke_with_context(s, "start")
self.invoke_with_context(
s,
"configure",
self.master.options,
self.master.options.keys()
)
def add(self, *addons):
"""
Add addons to the end of the chain, and run their startup events.
"""
if not addons:
raise ValueError("No addons specified.")
self.chain.extend(addons)
for i in addons:
self.invoke_with_context(i, "start")
self.invoke_with_context(
i,
"configure",
self.master.options,
self.master.options.keys()
)
self.startup(i)
def remove(self, addon):
"""
Remove an addon from the chain, and run its done events.
"""
self.chain = [i for i in self.chain if i is not addon]
self.invoke_with_context(addon, "done")

4
mitmproxy/builtins/filestreamer.py
View File

@ -40,11 +40,11 @@ class FileStreamer:
if err:
raise exceptions.OptionsError(err)
def tcp_open(self, flow):
def tcp_start(self, flow):
if self.stream:
self.active_flows.add(flow)
def tcp_close(self, flow):
def tcp_end(self, flow):
if self.stream:
self.stream.add(flow)
self.active_flows.discard(flow)

25
mitmproxy/builtins/script.py
View File

@ -88,6 +88,8 @@ def scriptenv(path, args):
sys.path.append(script_dir)
try:
yield
except SystemExit as v:
ctx.log.error("Script exited with code %s" % v.code)
except Exception:
etype, value, tb = sys.exc_info()
tb = cut_traceback(tb, "scriptenv").tb_next
@ -237,16 +239,35 @@ class ScriptLoader():
ctx.log.info("Un-loading script: %s" % a.name)
ctx.master.addons.remove(a)
# The machinations below are to ensure that:
# - Scripts remain in the same order
# - Scripts are listed directly after the script addon. This is
# needed to ensure that interactions with, for instance, flow
# serialization remains correct.
# - Scripts are not initialized un-necessarily. If only a
# script's order in the script list has changed, it should simply
# be moved.
current = {}
for a in ctx.master.addons.chain[:]:
if isinstance(a, Script):
current[a.name] = a
ctx.master.addons.chain.remove(a)
ordered = []
newscripts = []
for s in options.scripts:
if s in current:
ctx.master.addons.chain.append(current[s])
ordered.append(current[s])
else:
ctx.log.info("Loading script: %s" % s)
sc = Script(s)
ctx.master.addons.add(sc)
ordered.append(sc)
newscripts.append(sc)
ochain = ctx.master.addons.chain
pos = ochain.index(self)
ctx.master.addons.chain = ochain[:pos + 1] + ordered + ochain[pos + 1:]
for s in newscripts:
ctx.master.addons.startup(s)

25
mitmproxy/controller.py
View File

@ -19,10 +19,10 @@ Events = frozenset([
"serverconnect",
"serverdisconnect",
"tcp_open",
"tcp_start",
"tcp_message",
"tcp_error",
"tcp_close",
"tcp_end",
"request",
"requestheaders",
@ -49,24 +49,39 @@ class LogEntry(object):
class Log(object):
"""
The central logger, exposed to scripts as mitmproxy.ctx.log.
"""
def __init__(self, master):
self.master = master
def __call__(self, text, level="info"):
self.master.add_log(text, level)
def debug(self, txt):
"""
Log with level debug.
"""
self(txt, "debug")
def info(self, txt):
"""
Log with level info.
"""
self(txt, "info")
def warn(self, txt):
"""
Log with level warn.
"""
self(txt, "warn")
def error(self, txt):
"""
Log with level error.
"""
self(txt, "error")
def __call__(self, text, level="info"):
self.master.add_log(text, level)
class Master(object):
"""

2
mitmproxy/dump.py
View File

@ -37,9 +37,9 @@ class DumpMaster(flow.FlowMaster):
def __init__(self, server, options):
flow.FlowMaster.__init__(self, options, server, flow.DummyState())
self.has_errored = False
self.addons.add(termlog.TermLog())
self.addons.add(*builtins.default_addons())
self.addons.add(dumper.Dumper())
self.addons.add(termlog.TermLog())
# This line is just for type hinting
self.options = self.options # type: Options
self.set_stream_large_bodies(options.stream_large_bodies)

11
mitmproxy/flow/master.py
View File

@ -6,6 +6,7 @@ import sys
from typing import Optional # noqa
import netlib.exceptions
from netlib import http
from mitmproxy import controller
from mitmproxy import exceptions
from mitmproxy import models
@ -29,13 +30,13 @@ def event_sequence(f):
messages = f.messages
f.messages = []
f.reply = controller.DummyReply()
yield "tcp_open", f
yield "tcp_start", f
while messages:
f.messages.append(messages.pop(0))
yield "tcp_message", f
if f.error:
yield "tcp_error", f
yield "tcp_close", f
yield "tcp_end", f
else:
raise NotImplementedError
@ -83,7 +84,7 @@ class FlowMaster(controller.Master):
s = models.ServerConnection.make_dummy((host, port))
f = models.HTTPFlow(c, s)
headers = models.Headers()
headers = http.Headers()
req = models.HTTPRequest(
"absolute",
@ -261,7 +262,7 @@ class FlowMaster(controller.Master):
self.state.update_flow(f)
@controller.handler
def tcp_open(self, flow):
def tcp_start(self, flow):
# TODO: This would break mitmproxy currently.
# self.state.add_flow(flow)
pass
@ -275,5 +276,5 @@ class FlowMaster(controller.Master):
pass
@controller.handler
def tcp_close(self, flow):
def tcp_end(self, flow):
pass

4
mitmproxy/models/__init__.py
View File

@ -4,7 +4,7 @@ from netlib.http import decoded
from .connections import ClientConnection, ServerConnection
from .flow import Flow, Error
from .http import (
HTTPFlow, HTTPRequest, HTTPResponse, Headers,
HTTPFlow, HTTPRequest, HTTPResponse,
make_error_response, make_connect_request, make_connect_response, expect_continue_response
)
from .tcp import TCPFlow
@ -15,7 +15,7 @@ FLOW_TYPES = dict(
)
__all__ = [
"HTTPFlow", "HTTPRequest", "HTTPResponse", "Headers", "decoded",
"HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded",
"make_error_response", "make_connect_request",
"make_connect_response", "expect_continue_response",
"ClientConnection", "ServerConnection",

1
mitmproxy/models/flow.py
View File

@ -24,7 +24,6 @@ class Error(stateobject.StateObject):
Exposes the following attributes:
flow: Flow object
msg: Message describing the error
timestamp: Seconds since the epoch
"""

94
mitmproxy/models/http.py
View File

@ -1,37 +1,22 @@
from __future__ import absolute_import, print_function, division
import cgi
import warnings
from mitmproxy.models.flow import Flow
from mitmproxy.models import flow
from netlib import http
from netlib import version
from netlib.http import Headers
from netlib.http import Request
from netlib.http import Response
from netlib.http import status_codes
from netlib.tcp import Address
from netlib import tcp
class MessageMixin(object):
def get_decoded_content(self):
"""
Returns the decoded content based on the current Content-Encoding
header.
Doesn't change the message iteself or its headers.
"""
warnings.warn(".get_decoded_content() is deprecated, please use .content directly instead.", DeprecationWarning)
return self.content
class HTTPRequest(MessageMixin, Request):
class HTTPRequest(http.Request):
"""
A mitmproxy HTTP request.
This is a very thin wrapper on top of :py:class:`netlib.http.Request` and
may be removed in the future.
"""
# This is a very thin wrapper on top of :py:class:`netlib.http.Request` and
# may be removed in the future.
def __init__(
self,
first_line_format,
@ -49,7 +34,7 @@ class HTTPRequest(MessageMixin, Request):
stickycookie=False,
stickyauth=False,
):
Request.__init__(
http.Request.__init__(
self,
first_line_format,
method,
@ -110,13 +95,13 @@ class HTTPRequest(MessageMixin, Request):
return id(self)
class HTTPResponse(MessageMixin, Response):
class HTTPResponse(http.Response):
"""
A mitmproxy HTTP response.
This is a very thin wrapper on top of :py:class:`netlib.http.Response` and
may be removed in the future.
"""
# This is a very thin wrapper on top of :py:class:`netlib.http.Response` and
# may be removed in the future.
def __init__(
self,
@ -129,7 +114,7 @@ class HTTPResponse(MessageMixin, Response):
timestamp_end=None,
is_replay=False
):
Response.__init__(
http.Response.__init__(
self,
http_version,
status_code,
@ -161,34 +146,35 @@ class HTTPResponse(MessageMixin, Response):
return resp
class HTTPFlow(Flow):
class HTTPFlow(flow.Flow):
"""
A HTTPFlow is a collection of objects representing a single HTTP
An HTTPFlow is a collection of objects representing a single HTTP
transaction.
Attributes:
request: :py:class:`HTTPRequest` object
response: :py:class:`HTTPResponse` object
error: :py:class:`Error` object
server_conn: :py:class:`ServerConnection` object
client_conn: :py:class:`ClientConnection` object
intercepted: Is this flow currently being intercepted?
live: Does this flow have a live client connection?
Note that it's possible for a Flow to have both a response and an error
object. This might happen, for instance, when a response was received
from the server, but there was an error sending it back to the client.
"""
def __init__(self, client_conn, server_conn, live=None):
super(HTTPFlow, self).__init__("http", client_conn, server_conn, live)
self.request = None
"""@type: HTTPRequest"""
self.response = None
"""@type: HTTPResponse"""
_stateobject_attributes = Flow._stateobject_attributes.copy()
self.request = None
""" :py:class:`HTTPRequest` object """
self.response = None
""" :py:class:`HTTPResponse` object """
self.error = None
""" :py:class:`Error` object
Note that it's possible for a Flow to have both a response and an error
object. This might happen, for instance, when a response was received
from the server, but there was an error sending it back to the client.
"""
self.server_conn = server_conn
""" :py:class:`ServerConnection` object """
self.client_conn = client_conn
""":py:class:`ClientConnection` object """
self.intercepted = False
""" Is this flow currently being intercepted? """
_stateobject_attributes = flow.Flow._stateobject_attributes.copy()
_stateobject_attributes.update(
request=HTTPRequest,
response=HTTPResponse
@ -225,7 +211,7 @@ class HTTPFlow(Flow):
def make_error_response(status_code, message, headers=None):
response = status_codes.RESPONSES.get(status_code, "Unknown")
response = http.status_codes.RESPONSES.get(status_code, "Unknown")
body = """
<html>
<head>
@ -237,7 +223,7 @@ def make_error_response(status_code, message, headers=None):
body = body.encode("utf8", "replace")
if not headers:
headers = Headers(
headers = http.Headers(
Server=version.MITMPROXY,
Connection="close",
Content_Length=str(len(body)),
@ -254,10 +240,10 @@ def make_error_response(status_code, message, headers=None):
def make_connect_request(address):
address = Address.wrap(address)
address = tcp.Address.wrap(address)
return HTTPRequest(
"authority", b"CONNECT", None, address.host, address.port, None, b"HTTP/1.1",
Headers(), b""
http.Headers(), b""
)
@ -268,8 +254,10 @@ def make_connect_response(http_version):
http_version,
200,
b"Connection established",
Headers(),
http.Headers(),
b"",
)
expect_continue_response = HTTPResponse(b"HTTP/1.1", 100, b"Continue", Headers(), b"")
expect_continue_response = HTTPResponse(
b"HTTP/1.1", 100, b"Continue", http.Headers(), b""
)

4
mitmproxy/protocol/rawtcp.py
View File

@ -23,7 +23,7 @@ class RawTCPLayer(base.Layer):
if not self.ignore:
flow = models.TCPFlow(self.client_conn, self.server_conn, self)
self.channel.ask("tcp_open", flow)
self.channel.ask("tcp_start", flow)
buf = memoryview(bytearray(self.chunk_size))
@ -64,4 +64,4 @@ class RawTCPLayer(base.Layer):
self.channel.tell("tcp_error", flow)
finally:
if not self.ignore:
self.channel.tell("tcp_close", flow)
self.channel.tell("tcp_end", flow)

11
release/README.mkd
View File

@ -1,15 +1,4 @@
General build and release utilities for the mitmproxy, netlib and pathod
projects. These tools assume a directory structure with all repositories at the
same level, for example:
/src
/mitmproxy
/netlib
/pathod
/release
# Release policies
- By default, every release is a new minor (`0.x`) release and it will be

8
test/mitmproxy/builtins/test_script.py
View File

@ -237,12 +237,8 @@ class TestScriptLoader(mastertest.MasterTest):
"%s %s" % (rec, "b"),
]
debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"]
assert debug == [
('debug', 'c configure'),
('debug', 'a configure'),
('debug', 'b configure'),
]
m.event_log[:] = []
# No events, only order has changed
assert debug == []
o.scripts = [
"%s %s" % (rec, "x"),

2
test/mitmproxy/test_addons.py
View File

@ -16,6 +16,6 @@ def test_simple():
o = options.Options()
m = controller.Master(o)
a = addons.Addons(m)
a.add(o, TAddon("one"))
a.add(TAddon("one"))
assert a.get("one")
assert not a.get("two")

7
test/mitmproxy/test_examples.py
View File

@ -87,11 +87,8 @@ class TestScripts(mastertest.MasterTest):
m.request(f)
assert f.request.query["mitmproxy"] == "rocks"
def test_modify_response_body(self):
with tutils.raises(ScriptError):
tscript("modify_response_body.py")
m, sc = tscript("modify_response_body.py", "mitmproxy rocks")
def test_arguments(self):
m, sc = tscript("arguments.py", "mitmproxy rocks")
f = tutils.tflow(resp=netutils.tresp(content=b"I <3 mitmproxy"))
m.response(f)
assert f.response.content == b"I <3 rocks"