From 853cd810757a02252402da22ae77cab77c5a3fb6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 6 Sep 2015 01:37:15 +0200 Subject: [PATCH] docs++ --- docs/clientreplay.rst | 4 -- docs/conf.py | 3 +- docs/features/anticache.rst | 15 +++++++ docs/features/clientreplay.rst | 18 +++++++++ docs/features/filters.rst | 38 ++++++++++++++++++ docs/features/replacements.rst | 72 ++++++++++++++++++++++++++++++++++ docs/features/serverreplay.rst | 39 ++++++++++++++++++ docs/features/setheaders.rst | 18 +++++++++ docs/filters.rst | 4 -- docs/index.rst | 6 ++- setup.py | 4 +- 11 files changed, 210 insertions(+), 11 deletions(-) delete mode 100644 docs/clientreplay.rst create mode 100644 docs/features/anticache.rst create mode 100644 docs/features/clientreplay.rst create mode 100644 docs/features/filters.rst create mode 100644 docs/features/replacements.rst create mode 100644 docs/features/serverreplay.rst create mode 100644 docs/features/setheaders.rst delete mode 100644 docs/filters.rst diff --git a/docs/clientreplay.rst b/docs/clientreplay.rst deleted file mode 100644 index 4b9d720dc..000000000 --- a/docs/clientreplay.rst +++ /dev/null @@ -1,4 +0,0 @@ -.. _clientreplay: - -Client-side Replay -================== \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 1581b0556..1e6860079 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -36,7 +36,8 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.viewcode', - 'sphinx.ext.napoleon' + 'sphinx.ext.napoleon', + 'sphinxcontrib.documentedlist' ] autodoc_member_order = "bysource" diff --git a/docs/features/anticache.rst b/docs/features/anticache.rst new file mode 100644 index 000000000..5244587a1 --- /dev/null +++ b/docs/features/anticache.rst @@ -0,0 +1,15 @@ +.. _anticache: + +Anticache +========= +When the :option:`--anticache` option is passed to mitmproxy, it removes headers +(``if-none-match`` and ``if-modified-since``) that might elicit a +``304 not modified`` response from the server. This is useful when you want to make +sure you capture an HTTP exchange in its totality. It's also often used during +:ref:`clientreplay`, when you want to make sure the server responds with complete data. + + +================== ====================== +command-line :option:`--anticache` +mitmproxy shortcut :kbd:`o` then :kbd:`a` +================== ====================== \ No newline at end of file diff --git a/docs/features/clientreplay.rst b/docs/features/clientreplay.rst new file mode 100644 index 000000000..b8ca989ec --- /dev/null +++ b/docs/features/clientreplay.rst @@ -0,0 +1,18 @@ +.. _clientreplay: + +Client-side replay +================== + +Client-side replay does what it says on the tin: you provide a previously saved +HTTP conversation, and mitmproxy replays the client requests one by one. Note +that mitmproxy serializes the requests, waiting for a response from the server +before starting the next request. This might differ from the recorded +conversation, where requests may have been made concurrently. + +You may want to use client-side replay in conjunction with the +:ref:`anticache` option, to make sure the server responds with complete data. + +================== ================= +command-line :option:`-c path` +mitmproxy shortcut :kbd:`c` +================== ================= \ No newline at end of file diff --git a/docs/features/filters.rst b/docs/features/filters.rst new file mode 100644 index 000000000..5b22376c1 --- /dev/null +++ b/docs/features/filters.rst @@ -0,0 +1,38 @@ +.. _filters: + +Filter expressions +================== + +Many commands in :program:`mitmproxy` and :program:`mitmdump` take a filter expression. +Filter expressions consist of the following operators: + +.. documentedlist:: + :listobject: libmproxy.filt.help + +- Regexes are Python-style +- Regexes can be specified as quoted strings +- Header matching (~h, ~hq, ~hs) is against a string of the form "name: value". +- Strings with no operators are matched against the request URL. +- The default binary operator is &. + +Examples +-------- + +URL containing "google.com": + +.. code-block:: none + + google\.com + +Requests whose body contains the string "test": + +.. code-block:: none + + ~q ~b test + +Anything but requests with a text/html content type: + +.. code-block:: none + + !(~q & ~t "text/html") + diff --git a/docs/features/replacements.rst b/docs/features/replacements.rst new file mode 100644 index 000000000..8f7608664 --- /dev/null +++ b/docs/features/replacements.rst @@ -0,0 +1,72 @@ +.. _replacements: + +Replacements +============ + +Mitmproxy lets you specify an arbitrary number of patterns that define text +replacements within flows. Each pattern has 3 components: a filter that defines +which flows a replacement applies to, a regular expression that defines what +gets replaced, and a target value that defines what is substituted in. + +Replace hooks fire when either a client request or a server response is +received. Only the matching flow component is affected: so, for example, if a +replace hook is triggered on server response, the replacement is only run on +the Response object leaving the Request intact. You control whether the hook +triggers on the request, response or both using the filter pattern. If you need +finer-grained control than this, it's simple to create a script using the +replacement API on Flow components. + +Replacement hooks are extremely handy in interactive testing of applications. +For instance you can use a replace hook to replace the text "XSS" with a +complicated XSS exploit, and then "inject" the exploit simply by interacting +with the application through the browser. When used with tools like Firebug and +mitmproxy's own interception abilities, replacement hooks can be an amazingly +flexible and powerful feature. + + +On the command-line +------------------- + +The replacement hook command-line options use a compact syntax to make it easy +to specify all three components at once. The general form is as follows: + +.. code-block:: none + + /patt/regex/replacement + +Here, **patt** is a mitmproxy filter expression, **regex** is a valid Python +regular expression, and **replacement** is a string literal. The first +character in the expression (``/`` in this case) defines what the separation +character is. Here's an example of a valid expression that replaces "foo" with +"bar" in all requests: + +.. code-block:: none + + :~q:foo:bar + +In practice, it's pretty common for the replacement literal to be long and +complex. For instance, it might be an XSS exploit that weighs in at hundreds or +thousands of characters. To cope with this, there's a variation of the +replacement hook specifier that lets you load the replacement text from a file. +So, you might start **mitmdump** as follows: + +>>> mitmdump --replace-from-file :~q:foo:~/xss-exploit + +This will load the replacement text from the file ``~/xss-exploit``. + +Both the :option:`--replace` and :option:`--replace-from-file` flags can be passed multiple +times. + + +Interactively +------------- + +The :kbd:`R` shortcut key in the mitmproxy options menu (:kbd:`o`) lets you add and edit +replacement hooks using a built-in editor. The context-sensitive help (:kbd:`?`) has +complete usage information. + +================== ============================= +command-line :option:`--replace`, + :option:`--replace-from-file` +mitmproxy shortcut :kbd:`o` then :kbd:`R` +================== ============================= diff --git a/docs/features/serverreplay.rst b/docs/features/serverreplay.rst new file mode 100644 index 000000000..3b4af4e86 --- /dev/null +++ b/docs/features/serverreplay.rst @@ -0,0 +1,39 @@ +.. _serverreplay: + +Server-side replay +================== + +Server-side replay lets us replay server responses from a saved HTTP +conversation. + +Matching requests with responses +-------------------------------- + +By default, :program:`mitmproxy` excludes request headers when matching incoming +requests with responses from the replay file. This works in most circumstances, +and makes it possible to replay server responses in situations where request +headers would naturally vary, e.g. using a different user agent. +The :option:`--rheader headername` command-line option allows you to override +this behaviour by specifying individual headers that should be included in matching. + + +Response refreshing +------------------- + +Simply replaying server responses without modification will often result in +unexpected behaviour. For example cookie timeouts that were in the future at +the time a conversation was recorded might be in the past at the time it is +replayed. By default, :program:`mitmproxy` refreshes server responses before sending +them to the client. The **date**, **expires** and **last-modified** headers are +all updated to have the same relative time offset as they had at the time of +recording. So, if they were in the past at the time of recording, they will be +in the past at the time of replay, and vice versa. Cookie expiry times are +updated in a similar way. + +You can turn off response refreshing using the :option:`--norefresh` argument, or using +the :kbd:`o` options shortcut within :program:`mitmproxy`. + +================== ================= +command-line :option:`-S path` +mitmproxy shortcut :kbd:`S` +================== ================= \ No newline at end of file diff --git a/docs/features/setheaders.rst b/docs/features/setheaders.rst new file mode 100644 index 000000000..0a6c2296e --- /dev/null +++ b/docs/features/setheaders.rst @@ -0,0 +1,18 @@ +.. _setheaders: + +Set Headers +=========== + +This feature lets you specify a set of headers to be added to requests or +responses, based on a filter pattern. You can specify these either on the +command-line, or through an interactive editor in mitmproxy. + +Example: +.. code-block:: none + + mitmdump -R http://example.com --setheader :~q:Host:example.com + +================== ============================= +command-line :option:`--setheader PATTERN` +mitmproxy shortcut :kbd:`o` then :kbd:`H` +================== ============================= \ No newline at end of file diff --git a/docs/filters.rst b/docs/filters.rst deleted file mode 100644 index d23631500..000000000 --- a/docs/filters.rst +++ /dev/null @@ -1,4 +0,0 @@ -.. _filters: - -Filters -======= \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 33aa7b417..925830753 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,7 +23,11 @@ :hidden: :caption: Features - filters + features/anticache + features/filters + features/replacements + features/clientreplay + features/upstreamcerts .. toctree:: :hidden: diff --git a/setup.py b/setup.py index e28033ad1..6b2246a79 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,9 @@ dev_deps = { "nose-cov>=1.6", "coveralls>=0.4.1", "pathod>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION), - "countershape" + "sphinx>=1.3.1", + "sphinx-autobuild>=0.5.2", + "sphinxcontrib-documentedlist>=0.1", } # Add *all* script dependencies to developer dependencies. for script_deps in scripts.values():