From fdb6a44245249a50b5c95cdf0d8d13ecddfe5726 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 12 Oct 2016 10:57:05 +1300 Subject: [PATCH] docs: cleanups improvements and fighting sphinx - Hide links to internal code listings, and link to github instead - Improve formatting of code/example captions - Fix outdated documentation of command-line options - Complete documentation of all events + improved formatting - tcp_open -> tcp_start, tcp_close -> tcp_end to reduce confusion --- docs/_static/theme_overrides.css | 29 ++++ docs/conf.py | 45 +++++- docs/dev/models.rst | 81 ---------- docs/features/clientreplay.rst | 2 +- docs/features/responsestreaming.rst | 6 +- docs/features/serverreplay.rst | 2 +- docs/index.rst | 7 +- docs/scripting/api.rst | 8 + docs/scripting/context.rst | 4 + docs/scripting/events.rst | 202 +++++++++++++++++++++++++ docs/scripting/inlinescripts.rst | 227 ---------------------------- docs/scripting/mitmproxy.rst | 26 ---- docs/scripting/overview.rst | 79 ++++++++++ mitmproxy/builtins/filestreamer.py | 4 +- mitmproxy/controller.py | 4 +- mitmproxy/flow/master.py | 11 +- mitmproxy/models/__init__.py | 4 +- mitmproxy/protocol/rawtcp.py | 4 +- 18 files changed, 389 insertions(+), 356 deletions(-) delete mode 100644 docs/dev/models.rst create mode 100644 docs/scripting/api.rst create mode 100644 docs/scripting/context.rst create mode 100644 docs/scripting/events.rst delete mode 100644 docs/scripting/inlinescripts.rst delete mode 100644 docs/scripting/mitmproxy.rst create mode 100644 docs/scripting/overview.rst diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css index 585fdddb0..849f9f250 100644 --- a/docs/_static/theme_overrides.css +++ b/docs/_static/theme_overrides.css @@ -4,6 +4,10 @@ white-space: normal; } +.wy-table-responsive > table > tbody > tr > td { + vertical-align: top !important; +} + .wy-table-responsive { margin-bottom: 24px; max-width: 100%; @@ -13,3 +17,28 @@ .wy-menu-vertical header, .wy-menu-vertical p.caption { color: #e0e0e0; } + +.code-block-caption { + height: 1.5em; +} + +.code-block-caption .caption-text { + font-size: 0.8em; + float: right; +} + +.code-block-caption .headerlink { + display: none !important; +} + +.function .headerlink { + display: none !important; +} + +dl .reference.internal { + display: none !important; +} + +dl .headerlink { + display: none !important; +} diff --git a/docs/conf.py b/docs/conf.py index 54a353acd..76dc83d4d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,11 +1,16 @@ import sys import os +import importlib +import inspect sys.path.insert(0, os.path.abspath('..')) import netlib.version + extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', + 'sphinx.ext.extlinks', + 'sphinx.ext.linkcode', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', 'sphinxcontrib.documentedlist' @@ -156,7 +161,7 @@ html_static_path = ['_static'] #html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +html_show_sourcelink = False # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True @@ -189,5 +194,43 @@ html_static_path = ['_static'] # Output file base name for HTML help builder. htmlhelp_basename = 'mitmproxydoc' + +# FIXME: change master to dynamic version before release +extlinks = dict( + src = ('https://github.com/mitmproxy/mitmproxy/blob/master/%s', '') +) + + +MODULE = "/mitmproxy/" + +def linkcode_resolve(domain, info): + if domain != 'py': + return None + module, fullname = info['module'], info['fullname'] + # TODO: attributes/properties don't have modules, maybe try to look + # them up based on their cached host object? + if not module: + return None + obj = importlib.import_module(module) + for item in fullname.split('.'): + obj = getattr(obj, item, None) + if obj is None: + return None + try: + obj = getattr(obj, '_orig') + except AttributeError: + pass + try: + obj_source_path = inspect.getsourcefile(obj) + _, line = inspect.getsourcelines(obj) + except (TypeError, IOError): + # obj doesn't have a module, or something + return None + off = obj_source_path.rfind(MODULE) + mpath = obj_source_path[off + len(MODULE):] + print(obj_source_path, mpath) + return "https://github.com/mitmproxy/mitmproxy/blob/master/%s" % mpath + + def setup(app): app.add_stylesheet('theme_overrides.css') diff --git a/docs/dev/models.rst b/docs/dev/models.rst deleted file mode 100644 index a333fb06a..000000000 --- a/docs/dev/models.rst +++ /dev/null @@ -1,81 +0,0 @@ -.. _models: - -Datastructures -============== - -.. automodule:: mitmproxy.models - :members: HTTPFlow, HTTPRequest, HTTPResponse - - -.. automodule:: netlib.http - - .. autoclass:: Request - - .. rubric:: Data - .. autoattribute:: first_line_format - .. autoattribute:: method - .. autoattribute:: scheme - .. autoattribute:: host - .. autoattribute:: port - .. autoattribute:: path - .. autoattribute:: http_version - .. autoattribute:: headers - .. autoattribute:: content - .. autoattribute:: timestamp_start - .. autoattribute:: timestamp_end - .. rubric:: Computed Properties and Convenience Methods - .. autoattribute:: text - .. autoattribute:: url - .. autoattribute:: pretty_host - .. autoattribute:: pretty_url - .. autoattribute:: query - .. autoattribute:: cookies - .. autoattribute:: path_components - .. automethod:: anticache - .. automethod:: anticomp - .. automethod:: constrain_encoding - .. autoattribute:: urlencoded_form - .. autoattribute:: multipart_form - - .. autoclass:: Response - - .. automethod:: make - - .. rubric:: Data - .. autoattribute:: http_version - .. autoattribute:: status_code - .. autoattribute:: reason - .. autoattribute:: headers - .. autoattribute:: content - .. autoattribute:: timestamp_start - .. autoattribute:: timestamp_end - .. rubric:: Computed Properties and Convenience Methods - .. autoattribute:: text - .. autoattribute:: cookies - - .. autoclass:: Headers - :members: - :special-members: - :no-undoc-members: - -.. automodule:: netlib.multidict - - .. autoclass:: MultiDictView - - .. automethod:: get_all - .. automethod:: set_all - .. automethod:: add - .. automethod:: insert - .. automethod:: keys - .. automethod:: values - .. automethod:: items - .. automethod:: to_dict - -.. autoclass:: mitmproxy.models.Error - :show-inheritance: - -.. autoclass:: mitmproxy.models.ServerConnection - :show-inheritance: - -.. autoclass:: mitmproxy.models.ClientConnection - :show-inheritance: \ No newline at end of file diff --git a/docs/features/clientreplay.rst b/docs/features/clientreplay.rst index 50740bcfa..ebe40b5f8 100644 --- a/docs/features/clientreplay.rst +++ b/docs/features/clientreplay.rst @@ -14,5 +14,5 @@ You may want to use client-side replay in conjunction with the ================== =========== command-line ``-c path`` -mitmproxy shortcut :kbd:`c` +mitmproxy shortcut :kbd:`R` then :kbd:`c` ================== =========== diff --git a/docs/features/responsestreaming.rst b/docs/features/responsestreaming.rst index 66b5cae0e..1d5726c4e 100644 --- a/docs/features/responsestreaming.rst +++ b/docs/features/responsestreaming.rst @@ -35,10 +35,10 @@ command-line ``--stream SIZE`` Customizing Response Streaming ------------------------------ -You can also use an :ref:`inlinescripts` to customize exactly -which responses are streamed. +You can also use a script to customize exactly which responses are streamed. -Responses that should be tagged for streaming by setting their ``.stream`` attribute to ``True``: +Responses that should be tagged for streaming by setting their ``.stream`` +attribute to ``True``: .. literalinclude:: ../../examples/stream.py :caption: examples/stream.py diff --git a/docs/features/serverreplay.rst b/docs/features/serverreplay.rst index d70b6514e..f545d4a53 100644 --- a/docs/features/serverreplay.rst +++ b/docs/features/serverreplay.rst @@ -35,5 +35,5 @@ the :kbd:`o` options shortcut within :program:`mitmproxy`. ================== =========== command-line ``-S path`` -mitmproxy shortcut :kbd:`S` +mitmproxy shortcut :kbd:`R` then :kbd:`s` ================== =========== diff --git a/docs/index.rst b/docs/index.rst index 143f79f73..9a948678a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -51,9 +51,10 @@ :hidden: :caption: Scripting - scripting/inlinescripts - dev/models - scripting/mitmproxy + scripting/overview + scripting/context + scripting/events + scripting/api .. toctree:: diff --git a/docs/scripting/api.rst b/docs/scripting/api.rst new file mode 100644 index 000000000..03d359206 --- /dev/null +++ b/docs/scripting/api.rst @@ -0,0 +1,8 @@ +.. _api: + +API +==== + +.. automodule:: mitmproxy.models.http + :inherited-members: + :members: HTTPFlow, HTTPRequest, HTTPResponse diff --git a/docs/scripting/context.rst b/docs/scripting/context.rst new file mode 100644 index 000000000..7c3515983 --- /dev/null +++ b/docs/scripting/context.rst @@ -0,0 +1,4 @@ +.. _context: + +Context +======= diff --git a/docs/scripting/events.rst b/docs/scripting/events.rst new file mode 100644 index 000000000..c16c01f62 --- /dev/null +++ b/docs/scripting/events.rst @@ -0,0 +1,202 @@ +.. _events: + +Events +======= + +General +------- + +.. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - .. py:function:: configure(options, updated) + - Called once on startup, and whenever options change. + + *options* + An ``options.Options`` object with the total current configuration + state of mitmproxy. + *updated* + A set of strings indicating which configuration options have been + updated. This contains all options when *configure* is called on + startup, and only changed options subsequently. + + * - .. py:function:: done() + - Called once when the script shuts down, either because it's been + unloaded, or because the proxy itself is shutting down. + + * - .. py:function:: log(entry) + - Called whenever an event log is added. + + *entry* + An ``controller.LogEntry`` object - ``entry.msg`` is the log text, + and ``entry.level`` is the urgency level ("debug", "info", "warn", + "error"). + + * - .. py:function:: start() + - Called once on startup, before any other events. If you return a + value from this event, it will replace the current addon. This + allows you to, "boot into" an addon implemented as a class instance + from the module level. + + * - .. py:function:: tick() + - Called at a regular sub-second interval as long as the addon is + executing. + + +Connection +---------- + +.. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - .. py:function:: clientconnect(root_layer) + - Called when a client initiates a connection to the proxy. Note that a + connection can correspond to multiple HTTP requests. + + *root_layer* + The root layer (see `mitmproxy.protocol` for an explanation what + the root layer is), provides transparent access to all attributes + of the :py:class:`~mitmproxy.proxy.RootContext`. For example, + ``root_layer.client_conn.address`` gives the remote address of the + connecting client. + + * - .. py:function:: clientdisconnect(root_layer) + - Called when a client disconnects from the proxy. + + *root_layer* + The root layer object. + + * - .. py:function:: next_layer(layer) + + - Called whenever layers are switched. You may change which layer will + be used by returning a new layer object from this event. + + *layer* + The next layer, as determined by mitmpmroxy. + + * - .. py:function:: serverconnect(server_conn) + - Called before the proxy initiates a connection to the target server. + Note that a connection can correspond to multiple HTTP requests. + + *server_conn* + A ``ServerConnection`` object. It is guaranteed to have a non-None + ``address`` attribute. + + * - .. py:function:: serverdisconnect(server_conn) + - Called when the proxy has closed the server connection. + + *server_conn* + A ``ServerConnection`` object. + + +HTTP Events +----------- + +.. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - .. py:function:: request(flow) + - Called when a client request has been received. + + *flow* + A ``models.HTTPFlow`` object. At this point, the flow is + guaranteed to have a non-None ``request`` attribute. + + * - .. py:function:: requestheaders(flow) + - Called when the headers of a client request have been received, but + before the request body is read. + + *flow* + A ``models.HTTPFlow`` object. At this point, the flow is + guaranteed to have a non-None ``request`` attribute. + + * - .. py:function:: responseheaders(flow) + + - Called when the headers of a server response have been received, but + before the response body is read. + + *flow* + A ``models.HTTPFlow`` object. At this point, the flow is + guaranteed to have a non-none ``request`` and ``response`` + attributes, however the response will have no content. + + * - .. py:function:: response(flow) + + - Called when a server response has been received. + + *flow* + A ``models.HTTPFlow`` object. At this point, the flow is + guaranteed to have a non-none ``request`` and ``response`` + attributes. The raw response body will be in ``response.body``, + unless response streaming has been enabled. + + * - .. py:function:: error(flow) + - Called when a flow error has occurred, e.g. invalid server responses, + or interrupted connections. This is distinct from a valid server HTTP + error response, which is simply a response with an HTTP error code. + + *flow* + The flow containing the error. It is guaranteed to have + non-None ``error`` attribute. + + +WebSocket Events +----------------- + +.. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - .. py:function:: websockets_handshake(flow) + + - Called when a client wants to establish a WebSockets connection. The + WebSockets-specific headers can be manipulated to manipulate the + handshake. The ``flow`` object is guaranteed to have a non-None + ``request`` attribute. + + *flow* + The flow containing the HTTP websocket handshake request. The + object is guaranteed to have a non-None ``request`` attribute. + + +TCP Events +---------- + +These events are called only if the connection is in :ref:`TCP mode +`. So, for instance, TCP events are not called for ordinary HTTP/S +connections. + +.. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - .. py:function:: tcp_end(flow) + - Called when TCP streaming ends. + + *flow* + A ``models.TCPFlow`` object. + + * - .. py:function:: tcp_error(flow) + - Called when a TCP error occurs - e.g. the connection closing + unexpectedly. + + *flow* + A ``models.TCPFlow`` object. + + * - .. py:function:: tcp_message(flow) + + - Called a TCP payload is received from the client or server. The + sender and receiver are identifiable. The most recent message will be + ``flow.messages[-1]``. The message is user-modifiable. + + *flow* + A ``models.TCPFlow`` object. + + * - .. py:function:: tcp_start(flow) + - Called when TCP streaming starts. + + *flow* + A ``models.TCPFlow`` object. diff --git a/docs/scripting/inlinescripts.rst b/docs/scripting/inlinescripts.rst deleted file mode 100644 index 74d4e7144..000000000 --- a/docs/scripting/inlinescripts.rst +++ /dev/null @@ -1,227 +0,0 @@ -.. _inlinescripts: - -Inline Scripts -============== - -**mitmproxy** has a powerful scripting API that allows you to modify flows -on-the-fly or rewrite previously saved flows locally. - -The mitmproxy scripting API is event driven - a script is simply a Python -module that exposes a set of event methods. Here's a complete mitmproxy script -that adds a new header to every HTTP response before it is returned to the -client: - -.. literalinclude:: ../../examples/add_header.py - :caption: examples/add_header.py - :language: python - -All events that deal with an HTTP request get an instance of :py:class:`~mitmproxy.models.HTTPFlow`, -which we can use to manipulate the response itself. - -We can now run this script using mitmdump or mitmproxy as follows: - ->>> mitmdump -s add_header.py - -The new header will be added to all responses passing through the proxy. - -Examples --------- - -mitmproxy comes with a variety of example inline scripts, which demonstrate many basic tasks. -We encourage you to either browse them locally or on `GitHub`_. - - -Events ------- - -Script Lifecycle Events -^^^^^^^^^^^^^^^^^^^^^^^ - -.. py:function:: start(context) - - Called once on startup, before any other events. - - :param List[str] argv: The inline scripts' arguments. - For example, ``mitmproxy -s 'example.py --foo 42'`` sets argv to ``["--foo", "42"]``. - -.. py:function:: done(context) - - Called once on script shutdown, after any other events. - -Connection Events -^^^^^^^^^^^^^^^^^ - -.. py:function:: clientconnect(context, root_layer) - - Called when a client initiates a connection to the proxy. Note that - a connection can correspond to multiple HTTP requests. - - .. versionchanged:: 0.14 - - :param Layer root_layer: The root layer, which provides transparent access to all attributes of the - :py:class:`~mitmproxy.proxy.RootContext`. For example, ``root_layer.client_conn.address`` - gives the remote address of the connecting client. - -.. py:function:: clientdisconnect(context, root_layer) - - Called when a client disconnects from the proxy. - - .. versionchanged:: 0.14 - - :param Layer root_layer: see :py:func:`clientconnect` - -.. py:function:: serverconnect(context, server_conn) - - Called before the proxy initiates a connection to the target server. Note that - a connection can correspond to multiple HTTP requests. - - :param ServerConnection server_conn: The server connection object. It is guaranteed to have a - non-None ``address`` attribute. - -.. py:function:: serverdisconnect(context, server_conn) - - Called when the proxy has closed the server connection. - - .. versionadded:: 0.14 - - :param ServerConnection server_conn: see :py:func:`serverconnect` - -HTTP Events -^^^^^^^^^^^ - -.. py:function:: request(context, flow) - - Called when a client request has been received. The ``flow`` object is - guaranteed to have a non-None ``request`` attribute. - - :param HTTPFlow flow: The flow containing the request which has been received. - The object is guaranteed to have a non-None ``request`` attribute. - -.. py:function:: responseheaders(context, flow) - - Called when the headers of a server response have been received. - This will always be called before the response hook. - - :param HTTPFlow flow: The flow containing the request and response. - The object is guaranteed to have non-None ``request`` and - ``response`` attributes. ``response.content`` will be ``None``, - as the response body has not been read yet. - -.. py:function:: response(context, flow) - - Called when a server response has been received. - - :param HTTPFlow flow: The flow containing the request and response. - The object is guaranteed to have non-None ``request`` and - ``response`` attributes. ``response.body`` will contain the raw response body, - unless response streaming has been enabled. - -.. py:function:: error(context, flow) - - Called when a flow error has occurred, e.g. invalid server responses, or - interrupted connections. This is distinct from a valid server HTTP error - response, which is simply a response with an HTTP error code. - - :param HTTPFlow flow: The flow containing the error. - It is guaranteed to have non-None ``error`` attribute. - -WebSockets Events -^^^^^^^^^^^^^^^^^ - -.. py:function:: websocket_handshake(context, flow) - - Called when a client wants to establish a WebSockets connection. - The WebSockets-specific headers can be manipulated to manipulate the handshake. - The ``flow`` object is guaranteed to have a non-None ``request`` attribute. - - :param HTTPFlow flow: The flow containing the request which has been received. - The object is guaranteed to have a non-None ``request`` attribute. - -TCP Events -^^^^^^^^^^ - -.. py:function:: tcp_message(context, tcp_msg) - - .. warning:: API is subject to change - - If the proxy is in :ref:`TCP mode `, this event is called when it - receives a TCP payload from the client or server. - - The sender and receiver are identifiable. The message is user-modifiable. - - :param TcpMessage tcp_msg: see *examples/tcp_message.py* - -API ---- - -The canonical API documentation is the code, which you can browse here, locally or on `GitHub`_. -*Use the Source, Luke!* - -The main classes you will deal with in writing mitmproxy scripts are: - -:py:class:`mitmproxy.flow.FlowMaster` - - The "heart" of mitmproxy, usually subclassed as :py:class:`mitmproxy.dump.DumpMaster` or - :py:class:`mitmproxy.console.ConsoleMaster`. -:py:class:`~mitmproxy.models.ClientConnection` - - Describes a client connection. -:py:class:`~mitmproxy.models.ServerConnection` - - Describes a server connection. -:py:class:`~mitmproxy.models.HTTPFlow` - - A collection of objects representing a single HTTP transaction. -:py:class:`~mitmproxy.models.HTTPRequest` - - An HTTP request. -:py:class:`~mitmproxy.models.HTTPResponse` - - An HTTP response. -:py:class:`~mitmproxy.models.Error` - - A communications error. -:py:class:`netlib.http.Headers` - - A dictionary-like object for managing HTTP headers. -:py:class:`netlib.certutils.SSLCert` - - Exposes information SSL certificates. - - -Running scripts in parallel ---------------------------- - -We have a single flow primitive, so when a script is blocking, other requests are not processed. -While that's usually a very desirable behaviour, blocking scripts can be run threaded by using the -:py:obj:`mitmproxy.script.concurrent` decorator. -**If your script does not block, you should avoid the overhead of the decorator.** - -.. literalinclude:: ../../examples/nonblocking.py - :caption: examples/nonblocking.py - :language: python - -Make scripts configurable with arguments ----------------------------------------- - -Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by -surrounding the script call with quotes, e.g. ```mitmdump -s 'script.py --foo 42'``. -The arguments are then exposed in the start event: - -.. literalinclude:: ../../examples/modify_response_body.py - :caption: examples/modify_response_body.py - :language: python - -Running scripts on saved flows ------------------------------- - -Sometimes, we want to run a script on :py:class:`~mitmproxy.models.Flow` objects that are already -complete. This happens when you start a script, and then load a saved set of flows from a file -(see the "scripted data transformation" example :ref:`here `). -It also happens when you run a one-shot script on a single flow through the ``|`` (pipe) shortcut -in mitmproxy. - -In this case, there are no client connections, and the events are run in the following order: -**start**, **request**, **responseheaders**, **response**, **error**, **done**. -If the flow doesn't have a **response** or **error** associated with it, the matching events will -be skipped. - -Spaces in the script path -------------------------- - -By default, spaces are interpreted as a separator between the inline script and its arguments -(e.g. ``-s 'foo.py 42'``). Consequently, the script path needs to be wrapped in a separate pair of -quotes if it contains spaces: ``-s '\'./foo bar/baz.py\' 42'``. - -.. _GitHub: https://github.com/mitmproxy/mitmproxy diff --git a/docs/scripting/mitmproxy.rst b/docs/scripting/mitmproxy.rst deleted file mode 100644 index 9e3317360..000000000 --- a/docs/scripting/mitmproxy.rst +++ /dev/null @@ -1,26 +0,0 @@ - -FlowMaster -========== - -.. note:: - - We strongly encourage you to use :ref:`inlinescripts` rather than subclassing mitmproxy's FlowMaster. - - Inline Scripts are equally powerful and provide an easier syntax. - - Most examples are written as inline scripts. - - Multiple inline scripts can be used together. - - Inline Scripts can either be executed headless with mitmdump or within the mitmproxy UI. - - -All of mitmproxy's basic functionality is exposed through the **mitmproxy** -library. The example below shows a simple implementation of the "sticky cookie" -functionality included in the interactive mitmproxy program. Traffic is -monitored for ``Cookie`` and ``Set-Cookie`` headers, and requests are rewritten -to include a previously seen cookie if they don't already have one. In effect, -this lets you log in to a site using your browser, and then make subsequent -requests using a tool like curl, which will then seem to be part of the -authenticated session. - - -.. literalinclude:: ../../examples/stickycookies - :caption: examples/stickycookies - :language: python diff --git a/docs/scripting/overview.rst b/docs/scripting/overview.rst new file mode 100644 index 000000000..a0dfe111c --- /dev/null +++ b/docs/scripting/overview.rst @@ -0,0 +1,79 @@ +.. _overview: + +Overview +========= + +Mitmproxy has a powerful scripting API that allows you to control almost any +aspect of traffic being proxied. In fact, much of mitmproxy's own core +functionality is implemented using the exact same API exposed to scripters (see +:src:`mitmproxy/builtins`). + +Scripting is event driven, with named handlers on the script object called at +appropriate points of mitmproxy's operation. Here's a complete mitmproxy script +that adds a new header to every HTTP response before it is returned to the +client: + +.. literalinclude:: ../../examples/add_header.py + :caption: :src:`examples/add_header.py` + :language: python + +All events that deal with an HTTP request get an instance of +:py:class:`~mitmproxy.models.HTTPFlow`, which we can use to manipulate the +response itself. We can now run this script using mitmdump or mitmproxy as +follows: + +>>> mitmdump -s add_header.py + +The new header will be added to all responses passing through the proxy. + + +mitmproxy comes with a variety of example inline scripts, which demonstrate +many basic tasks. + + +Running scripts in parallel +--------------------------- + +We have a single flow primitive, so when a script is blocking, other requests are not processed. +While that's usually a very desirable behaviour, blocking scripts can be run threaded by using the +:py:obj:`mitmproxy.script.concurrent` decorator. +**If your script does not block, you should avoid the overhead of the decorator.** + +.. literalinclude:: ../../examples/nonblocking.py + :caption: examples/nonblocking.py + :language: python + +Make scripts configurable with arguments +---------------------------------------- + +Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by +surrounding the script call with quotes, e.g. ```mitmdump -s 'script.py --foo 42'``. +The arguments are then exposed in the start event: + +.. literalinclude:: ../../examples/modify_response_body.py + :caption: examples/modify_response_body.py + :language: python + + +Running scripts on saved flows +------------------------------ + +Sometimes, we want to run a script on :py:class:`~mitmproxy.models.Flow` objects that are already +complete. This happens when you start a script, and then load a saved set of flows from a file +(see the "scripted data transformation" example :ref:`here `). +It also happens when you run a one-shot script on a single flow through the ``|`` (pipe) shortcut +in mitmproxy. + +In this case, there are no client connections, and the events are run in the following order: +**start**, **request**, **responseheaders**, **response**, **error**, **done**. +If the flow doesn't have a **response** or **error** associated with it, the matching events will +be skipped. + +Spaces in the script path +------------------------- + +By default, spaces are interpreted as a separator between the inline script and its arguments +(e.g. ``-s 'foo.py 42'``). Consequently, the script path needs to be wrapped in a separate pair of +quotes if it contains spaces: ``-s '\'./foo bar/baz.py\' 42'``. + +.. _GitHub: https://github.com/mitmproxy/mitmproxy diff --git a/mitmproxy/builtins/filestreamer.py b/mitmproxy/builtins/filestreamer.py index 3436e0760..bf5eedee4 100644 --- a/mitmproxy/builtins/filestreamer.py +++ b/mitmproxy/builtins/filestreamer.py @@ -40,11 +40,11 @@ class FileStreamer: if err: raise exceptions.OptionsError(err) - def tcp_open(self, flow): + def tcp_start(self, flow): if self.stream: self.active_flows.add(flow) - def tcp_close(self, flow): + def tcp_end(self, flow): if self.stream: self.stream.add(flow) self.active_flows.discard(flow) diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 1a8801e1f..7b9d460ac 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -19,10 +19,10 @@ Events = frozenset([ "serverconnect", "serverdisconnect", - "tcp_open", + "tcp_start", "tcp_message", "tcp_error", - "tcp_close", + "tcp_end", "request", "requestheaders", diff --git a/mitmproxy/flow/master.py b/mitmproxy/flow/master.py index a2b225b86..a1b0a6311 100644 --- a/mitmproxy/flow/master.py +++ b/mitmproxy/flow/master.py @@ -6,6 +6,7 @@ import sys from typing import Optional # noqa import netlib.exceptions +from netlib import http from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import models @@ -29,13 +30,13 @@ def event_sequence(f): messages = f.messages f.messages = [] f.reply = controller.DummyReply() - yield "tcp_open", f + yield "tcp_start", f while messages: f.messages.append(messages.pop(0)) yield "tcp_message", f if f.error: yield "tcp_error", f - yield "tcp_close", f + yield "tcp_end", f else: raise NotImplementedError @@ -83,7 +84,7 @@ class FlowMaster(controller.Master): s = models.ServerConnection.make_dummy((host, port)) f = models.HTTPFlow(c, s) - headers = models.Headers() + headers = http.Headers() req = models.HTTPRequest( "absolute", @@ -261,7 +262,7 @@ class FlowMaster(controller.Master): self.state.update_flow(f) @controller.handler - def tcp_open(self, flow): + def tcp_start(self, flow): # TODO: This would break mitmproxy currently. # self.state.add_flow(flow) pass @@ -275,5 +276,5 @@ class FlowMaster(controller.Master): pass @controller.handler - def tcp_close(self, flow): + def tcp_end(self, flow): pass diff --git a/mitmproxy/models/__init__.py b/mitmproxy/models/__init__.py index 9bd197233..7b7eaef0c 100644 --- a/mitmproxy/models/__init__.py +++ b/mitmproxy/models/__init__.py @@ -4,7 +4,7 @@ from netlib.http import decoded from .connections import ClientConnection, ServerConnection from .flow import Flow, Error from .http import ( - HTTPFlow, HTTPRequest, HTTPResponse, Headers, + HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_request, make_connect_response, expect_continue_response ) from .tcp import TCPFlow @@ -15,7 +15,7 @@ FLOW_TYPES = dict( ) __all__ = [ - "HTTPFlow", "HTTPRequest", "HTTPResponse", "Headers", "decoded", + "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded", "make_error_response", "make_connect_request", "make_connect_response", "expect_continue_response", "ClientConnection", "ServerConnection", diff --git a/mitmproxy/protocol/rawtcp.py b/mitmproxy/protocol/rawtcp.py index 70486cc42..069420ea6 100644 --- a/mitmproxy/protocol/rawtcp.py +++ b/mitmproxy/protocol/rawtcp.py @@ -23,7 +23,7 @@ class RawTCPLayer(base.Layer): if not self.ignore: flow = models.TCPFlow(self.client_conn, self.server_conn, self) - self.channel.ask("tcp_open", flow) + self.channel.ask("tcp_start", flow) buf = memoryview(bytearray(self.chunk_size)) @@ -64,4 +64,4 @@ class RawTCPLayer(base.Layer): self.channel.tell("tcp_error", flow) finally: if not self.ignore: - self.channel.tell("tcp_close", flow) + self.channel.tell("tcp_end", flow)