Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-22 07:08:10 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4422 from mhils/pdoc

Browse Source 
Docs: Add API Reference Using Pdoc
This commit is contained in:
Maximilian Hils 2021-02-13 00:18:49 +01:00 committed by GitHub
commit 748fc93699
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
50 changed files with 1043 additions and 695 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs/build.py Executable file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env python3
import shutil
import subprocess
from pathlib import Path
here = Path(__file__).parent
for script in (here / "scripts").glob("*.py"):
print(f"Generating output for {script.name}...")
out = subprocess.check_output(["python3", script.absolute()], text=True)
if out:
(here / "src" / "generated" / f"{script.stem}.html").write_text(out, encoding="utf8")
if (here / "public").exists():
shutil.rmtree(here / "public")
subprocess.run(["hugo"], cwd=here / "src")

20
docs/build.sh
View File

@ -1,20 +0,0 @@
#!/usr/bin/env bash
set -o errexit
set -o pipefail
set -o nounset
# set -o xtrace
SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
pushd ${SCRIPTPATH}
for script in scripts/*.py ; do
output="${script##*/}"
output="src/generated/${output%.*}.html"
echo "Generating output for ${script} into ${output} ..."
"${script}" > "${output}"
done
rm -rf ./public
cd src
hugo

2
docs/ci.sh
View File

@ -6,7 +6,7 @@ set -o pipefail
# This script gets run from CI to render and upload docs for the master branch.
./build.sh
./build.py
# Only upload if we have defined credentials - we only have these defined for
# trusted commits (i.e. not PRs).

4
docs/modd.conf
View File

@ -1,5 +1,5 @@
scripts/*.py {
prep: ./build.sh
scripts/** {
prep: python3 build.py
}
{

151
docs/scripts/api-events.py Normal file
View File

@ -0,0 +1,151 @@
#!/usr/bin/env python3
import contextlib
import inspect
import textwrap
from pathlib import Path
from typing import List, Type
import mitmproxy.addons.next_layer # noqa
from mitmproxy import hooks, log, addonmanager
from mitmproxy.proxy import server_hooks, layer
from mitmproxy.proxy.layers import http, tcp, tls, websocket
known = set()
def category(name: str, desc: str, hooks: List[Type[hooks.Hook]]) -> None:
all_params = [
list(inspect.signature(hook.__init__).parameters.values())[1:]
for hook in hooks
]
# slightly overengineered, but this was fun to write. ¯\_(ツ)_/¯
imports = set()
types = set()
for params in all_params:
for param in params:
try:
mod = inspect.getmodule(param.annotation).__name__
if mod == "typing":
# this is ugly, but can be removed once we are on Python 3.9+ only
imports.add(inspect.getmodule(param.annotation.__args__[0]).__name__)
types.add(param.annotation._name)
else:
imports.add(mod)
except AttributeError:
raise ValueError(f"Missing type annotation: {params}")
imports.discard("builtins")
if types:
print(f"from typing import {', '.join(sorted(types))}")
print("from mitmproxy import ctx")
for imp in sorted(imports):
print(f"import {imp}")
print()
print(f"class {name}Events:")
print(f' """{desc}"""')
first = True
for hook, params in zip(hooks, all_params):
if first:
first = False
else:
print()
if hook.name in known:
raise RuntimeError(f"Already documented: {hook}")
known.add(hook.name)
doc = inspect.getdoc(hook)
print(f" def {hook.name}({', '.join(str(p) for p in ['self'] + params)}):")
print(textwrap.indent(f'"""\n{doc}\n"""', " "))
if params:
print(f' ctx.log(f"{hook.name}: {" ".join("{" + p.name + "=}" for p in params)}")')
else:
print(f' ctx.log("{hook.name}")')
print("")
outfile = Path(__file__).parent.parent / "src" / "generated" / "events.py"
with outfile.open("w") as f, contextlib.redirect_stdout(f):
print("# This file is autogenerated, do not edit manually.")
category(
"Lifecycle",
"",
[
addonmanager.LoadHook,
hooks.RunningHook,
hooks.ConfigureHook,
hooks.DoneHook,
]
)
category(
"Connection",
"",
[
server_hooks.ClientConnectedHook,
server_hooks.ClientDisconnectedHook,
server_hooks.ServerConnectHook,
server_hooks.ServerConnectedHook,
server_hooks.ServerDisconnectedHook,
]
)
category(
"HTTP",
"",
[
http.HttpRequestHeadersHook,
http.HttpRequestHook,
http.HttpResponseHeadersHook,
http.HttpResponseHook,
http.HttpErrorHook,
http.HttpConnectHook,
]
)
category(
"TCP",
"",
[
tcp.TcpStartHook,
tcp.TcpMessageHook,
tcp.TcpEndHook,
tcp.TcpErrorHook,
]
)
category(
"TLS",
"",
[
tls.TlsClienthelloHook,
tls.TlsStartHook,
]
)
category(
"WebSocket",
"",
[
websocket.WebsocketStartHook,
websocket.WebsocketMessageHook,
websocket.WebsocketEndHook,
websocket.WebsocketErrorHook,
]
)
category(
"AdvancedLifecycle",
"",
[
layer.NextLayerHook,
hooks.UpdateHook,
log.AddLogHook,
]
)
not_documented = set(hooks.all_hooks.keys()) - known
if not_documented:
raise RuntimeError(f"Not documented: {not_documented}")

67
docs/scripts/api-render.py Normal file
View File

@ -0,0 +1,67 @@
#!/usr/bin/env python3
import os
import shutil
import textwrap
from pathlib import Path
import pdoc.render_helpers
here = Path(__file__).parent
if os.environ.get("DOCS_ARCHIVE", False):
edit_url_map = {}
else:
edit_url_map = {
"mitmproxy": "https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/",
}
pdoc.render.configure(
template_directory=here / "pdoc-template",
edit_url_map=edit_url_map,
)
# We can't configure Hugo, but we can configure pdoc.
pdoc.render_helpers.formatter.cssclass = "chroma"
modules = [
"mitmproxy.addonmanager",
"mitmproxy.certs",
"mitmproxy.connection",
"mitmproxy.coretypes.multidict",
"mitmproxy.flow",
"mitmproxy.http",
"mitmproxy.net.server_spec",
"mitmproxy.proxy.server_hooks",
"mitmproxy.tcp",
"mitmproxy.websocket",
here / ".." / "src" / "generated" / "events.py",
]
pdoc.pdoc(
*modules,
output_directory=here / ".." / "src" / "generated" / "api"
)
api_content = here / ".." / "src" / "content" / "api"
if api_content.exists():
shutil.rmtree(api_content)
api_content.mkdir()
for module in modules:
if isinstance(module, Path):
continue
filename = f"api/{module.replace('.', '/')}.html"
(api_content / f"{module}.md").write_text(textwrap.dedent(f"""
---
title: "{module}"
url: "{filename}"
menu:
addons:
parent: 'Event Hooks & API'
---
{{{{< readfile file="/generated/{filename}" >}}}}
"""))
(here / ".." / "src" / "content" / "addons-api.md").touch()

142
docs/scripts/events.py
View File

@ -1,142 +0,0 @@
#!/usr/bin/env python3
import inspect
import textwrap
from typing import List, Type
import mitmproxy.addons.next_layer # noqa
from mitmproxy import hooks, log, addonmanager
from mitmproxy.proxy import server_hooks, layer
from mitmproxy.proxy.layers import http, tcp, tls, websocket
known = set()
def category(name: str, hooks: List[Type[hooks.Hook]]) -> None:
print(f"### {name} Events")
print("```python")
all_params = [
list(inspect.signature(hook.__init__).parameters.values())[1:]
for hook in hooks
]
# slightly overengineered, but this was fun to write. ¯\_(ツ)_/¯
imports = set()
types = set()
for params in all_params:
for param in params:
try:
mod = inspect.getmodule(param.annotation).__name__
if mod == "typing":
# this is ugly, but can be removed once we are on Python 3.9+ only
imports.add(inspect.getmodule(param.annotation.__args__[0]).__name__)
types.add(param.annotation._name)
else:
imports.add(mod)
except AttributeError:
raise ValueError(f"Missing type annotation: {params}")
imports.discard("builtins")
if types:
print(f"from typing import {', '.join(sorted(types))}")
print("from mitmproxy import ctx")
for imp in sorted(imports):
print(f"import {imp}")
print()
first = True
for hook, params in zip(hooks, all_params):
if first:
first = False
else:
print()
if hook.name in known:
raise RuntimeError(f"Already documented: {hook}")
known.add(hook.name)
doc = inspect.getdoc(hook)
print(f"def {hook.name}({', '.join(str(p) for p in params)}):")
print(textwrap.indent(f'"""\n{doc}\n"""', " "))
if params:
print(f' ctx.log(f"{hook.name}: {" ".join("{" + p.name + "=}" for p in params)}")')
else:
print(f' ctx.log("{hook.name}")')
print("```")
category(
"Lifecycle",
[
addonmanager.LoadHook,
hooks.RunningHook,
hooks.ConfigureHook,
hooks.DoneHook,
]
)
category(
"Connection",
[
server_hooks.ClientConnectedHook,
server_hooks.ClientDisconnectedHook,
server_hooks.ServerConnectHook,
server_hooks.ServerConnectedHook,
server_hooks.ServerDisconnectedHook,
]
)
category(
"HTTP",
[
http.HttpRequestHeadersHook,
http.HttpRequestHook,
http.HttpResponseHeadersHook,
http.HttpResponseHook,
http.HttpErrorHook,
http.HttpConnectHook,
]
)
category(
"TCP",
[
tcp.TcpStartHook,
tcp.TcpMessageHook,
tcp.TcpEndHook,
tcp.TcpErrorHook,
]
)
category(
"TLS",
[
tls.TlsClienthelloHook,
tls.TlsStartHook,
]
)
category(
"WebSocket",
[
websocket.WebsocketStartHook,
websocket.WebsocketMessageHook,
websocket.WebsocketEndHook,
websocket.WebsocketErrorHook,
]
)
category(
"Advanced Lifecycle",
[
layer.NextLayerHook,
hooks.UpdateHook,
log.AddLogHook,
]
)
not_documented = set(hooks.all_hooks.keys()) - known
if not_documented:
raise RuntimeError(f"Not documented: {not_documented}")
# print("<table class=\"table filtertable\"><tbody>")
# for i in flowfilter.help:
# print("<tr><th>%s</th><td>%s</td></tr>" % i)
# print("</tbody></table>")

31
docs/scripts/examples.py
View File

@ -28,21 +28,38 @@ for example in examples:
else:
comment = ""
overview.append(
f" * [{example.name}](#{slug}){comment}"
f" * [{example.name}](#{slug}){comment}\n"
)
listings.append(f"""
<h2 id="{slug}">Example: {example.name}</h2>
<h3 id="{slug}">Example: {example.name}</h3>
```python
{code}
{code.strip()}
```
""")
print("\n".join(overview))
print("""
### Community Examples
print(f"""
# Addon Examples
### Dedicated Example Addons
{"".join(overview)}
### Built-In Addons
Much of mitmproxy’s own functionality is defined in
[a suite of built-in addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons),
implementing everything from functionality like anticaching and sticky cookies to our onboarding webapp.
The built-in addons make for instructive reading, and you will quickly see that quite complex functionality
can often boil down to a very small, completely self-contained modules.
### Additional Community Examples
Additional examples contributed by the mitmproxy community can be found
[on GitHub](https://github.com/mitmproxy/mitmproxy/tree/master/examples/contrib).
-------------------------
{"".join(listings)}
""")
print("\n".join(listings))

3
docs/scripts/pdoc-template/frame.html.jinja2 Normal file
View File

@ -0,0 +1,3 @@
{% block style %}{% endblock %}
{% block body %}{% endblock %}
<div class="pdoc" style="margin-top: 4rem">{% block attribution %}{% endblock %}</div>

62
docs/scripts/pdoc-template/module.html.jinja2 Normal file
View File

@ -0,0 +1,62 @@
{% extends "default/module.html.jinja2" %}
{% block nav %}{% endblock %}
{% block style_layout %}{% endblock %}
{% block style_pygments %}{% endblock %}
{#
To document all event hooks, we do a bit of hackery:
1. scripts/api-events.py auto-generates generated/events.py.
2. scripts/api-render.py renders generated/events.py together with the remaining API docs.
3. This templates hides some elements of the default pdoc template.
#}
{% if module.name == "events" %}
{% macro module_name() %}
{% endmacro %}
{% macro view_source(doc) %}
{% if doc.type != "module" %}
{{ default_view_source(doc) }}
{% endif %}
{% endmacro %}
{% macro is_public(doc) %}
{% if doc.name != "__init__" %}
{{ default_is_public(doc) }}
{% endif %}
{% endmacro %}
{% else %}
{% macro is_public(doc) %}
{% if doc.name is in(["from_state", "get_state", "set_state"]) %}
{% elif doc.modulename == "mitmproxy.addonmanager" %}
{% if doc.qualname.startswith("Loader") and not doc.name.startswith("_") %}
true
{% endif %}
{% elif doc.modulename == "mitmproxy.certs" %}
{% if doc.qualname == "Cert" or doc.qualname.startswith("Cert.") %}
{{ default_is_public(doc) }}
{% endif %}
{% elif doc.modulename == "mitmproxy.coretypes.multidict" %}
{% if doc.name == "_MultiDict" %}
true
{% else %}
{{ default_is_public(doc) }}
{% endif %}
{% elif doc.modulename == "mitmproxy.flow" %}
{% if doc.name is not in(["__init__", "reply", "metadata"]) %}
{{ default_is_public(doc) }}
{% endif %}
{% elif doc.modulename == "mitmproxy.http" %}
{% if doc.qualname is not in([
"Message.__init__", "Message.data",
"Request.data",
"Response.data",
]) %}
{{ default_is_public(doc) }}
{% endif %}
{% elif doc.modulename == "mitmproxy.proxy.server_hooks" %}
{% if doc.qualname.startswith("ServerConnectionHookData") and doc.name != "__init__" %}
{{ default_is_public(doc) }}
{% endif %}
{% else %}
{{ default_is_public(doc) }}
{% endif %}
{% endmacro %}
{% endif %}

15
docs/src/assets/style.scss
View File

@ -1,4 +1,12 @@
@import "./syntax";
/* background for both hugo *and* pdoc. */
.chroma pre, pre.chroma {
background-color: #f7f7f7;
border-top: 1px solid #ccc;
border-bottom: 1px solid #ccc;
padding: .5rem 0 .5rem .5rem;
}
@import "./badge";
$primary: #C93312;
@ -7,6 +15,9 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox
$panel-heading-size: 1em;
$panel-heading-weight: 600;
$menu-list-link-padding: .3em .75em;
$menu-label-spacing: .7em;
$menu-nested-list-margin: .3em .75em;
/*!*
bulma.io v0.8.0 | MIT License | github.com/jgthms/bulma */
@ -17,6 +28,10 @@ bulma.io v0.8.0 | MIT License | github.com/jgthms/bulma */
@import "./bulma/components/_all";
@import "./bulma/layout/_all";
html {
scroll-behavior: smooth;
}
html, body {
height: 100%;
}

135
docs/src/assets/syntax.css
View File

@ -1,59 +1,82 @@
/* Background */ .chroma { color: #f8f8f2; background-color: #272822 }
/* Error */ .chroma .err { color: #960050; background-color: #1e0010 }
/* Background */ .chroma { background-color: #ffffff }
/* Other */ .chroma .x { }
/* Error */ .chroma .err { color: #a61717; background-color: #e3d2d2 }
/* LineTableTD */ .chroma .lntd { vertical-align: top; padding: 0; margin: 0; border: 0; }
/* LineTable */ .chroma .lntable { border-spacing: 0; padding: 0; margin: 0; border: 0; width: 100%; overflow: auto; display: block; }
/* LineTable */ .chroma .lntable { border-spacing: 0; padding: 0; margin: 0; border: 0; width: auto; overflow: auto; display: block; }
/* LineHighlight */ .chroma .hl { display: block; width: 100%;background-color: #ffffcc }
/* LineNumbersTable */ .chroma .lnt { margin-right: 0.4em; padding: 0 0.4em 0 0.4em; display: block; }
/* LineNumbers */ .chroma .ln { margin-right: 0.4em; padding: 0 0.4em 0 0.4em; }
/* Keyword */ .chroma .k { color: #66d9ef }
/* KeywordConstant */ .chroma .kc { color: #66d9ef }
/* KeywordDeclaration */ .chroma .kd { color: #66d9ef }
/* KeywordNamespace */ .chroma .kn { color: #f92672 }
/* KeywordPseudo */ .chroma .kp { color: #66d9ef }
/* KeywordReserved */ .chroma .kr { color: #66d9ef }
/* KeywordType */ .chroma .kt { color: #66d9ef }
/* NameAttribute */ .chroma .na { color: #a6e22e }
/* NameClass */ .chroma .nc { color: #a6e22e }
/* NameConstant */ .chroma .no { color: #66d9ef }
/* NameDecorator */ .chroma .nd { color: #a6e22e }
/* NameException */ .chroma .ne { color: #a6e22e }
/* NameFunction */ .chroma .nf { color: #a6e22e }
/* NameOther */ .chroma .nx { color: #a6e22e }
/* NameTag */ .chroma .nt { color: #f92672 }
/* Literal */ .chroma .l { color: #ae81ff }
/* LiteralDate */ .chroma .ld { color: #e6db74 }
/* LiteralString */ .chroma .s { color: #e6db74 }
/* LiteralStringAffix */ .chroma .sa { color: #e6db74 }
/* LiteralStringBacktick */ .chroma .sb { color: #e6db74 }
/* LiteralStringChar */ .chroma .sc { color: #e6db74 }
/* LiteralStringDelimiter */ .chroma .dl { color: #e6db74 }
/* LiteralStringDoc */ .chroma .sd { color: #e6db74 }
/* LiteralStringDouble */ .chroma .s2 { color: #e6db74 }
/* LiteralStringEscape */ .chroma .se { color: #ae81ff }
/* LiteralStringHeredoc */ .chroma .sh { color: #e6db74 }
/* LiteralStringInterpol */ .chroma .si { color: #e6db74 }
/* LiteralStringOther */ .chroma .sx { color: #e6db74 }
/* LiteralStringRegex */ .chroma .sr { color: #e6db74 }
/* LiteralStringSingle */ .chroma .s1 { color: #e6db74 }
/* LiteralStringSymbol */ .chroma .ss { color: #e6db74 }
/* LiteralNumber */ .chroma .m { color: #ae81ff }
/* LiteralNumberBin */ .chroma .mb { color: #ae81ff }
/* LiteralNumberFloat */ .chroma .mf { color: #ae81ff }
/* LiteralNumberHex */ .chroma .mh { color: #ae81ff }
/* LiteralNumberInteger */ .chroma .mi { color: #ae81ff }
/* LiteralNumberIntegerLong */ .chroma .il { color: #ae81ff }
/* LiteralNumberOct */ .chroma .mo { color: #ae81ff }
/* Operator */ .chroma .o { color: #f92672 }
/* OperatorWord */ .chroma .ow { color: #f92672 }
/* Comment */ .chroma .c { color: #75715e }
/* CommentHashbang */ .chroma .ch { color: #75715e }
/* CommentMultiline */ .chroma .cm { color: #75715e }
/* CommentSingle */ .chroma .c1 { color: #75715e }
/* CommentSpecial */ .chroma .cs { color: #75715e }
/* CommentPreproc */ .chroma .cp { color: #75715e }
/* CommentPreprocFile */ .chroma .cpf { color: #75715e }
/* GenericDeleted */ .chroma .gd { color: #f92672 }
/* GenericEmph */ .chroma .ge { font-style: italic }
/* GenericInserted */ .chroma .gi { color: #a6e22e }
/* LineNumbersTable */ .chroma .lnt { margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #7f7f7f }
/* LineNumbers */ .chroma .ln { margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #7f7f7f }
/* Keyword */ .chroma .k { color: #000000; font-weight: bold }
/* KeywordConstant */ .chroma .kc { color: #000000; font-weight: bold }
/* KeywordDeclaration */ .chroma .kd { color: #000000; font-weight: bold }
/* KeywordNamespace */ .chroma .kn { color: #000000; font-weight: bold }
/* KeywordPseudo */ .chroma .kp { color: #000000; font-weight: bold }
/* KeywordReserved */ .chroma .kr { color: #000000; font-weight: bold }
/* KeywordType */ .chroma .kt { color: #445588; font-weight: bold }
/* Name */ .chroma .n { }
/* NameAttribute */ .chroma .na { color: #008080 }
/* NameBuiltin */ .chroma .nb { color: #0086b3 }
/* NameBuiltinPseudo */ .chroma .bp { color: #999999 }
/* NameClass */ .chroma .nc { color: #445588; font-weight: bold }
/* NameConstant */ .chroma .no { color: #008080 }
/* NameDecorator */ .chroma .nd { color: #3c5d5d; font-weight: bold }
/* NameEntity */ .chroma .ni { color: #800080 }
/* NameException */ .chroma .ne { color: #990000; font-weight: bold }
/* NameFunction */ .chroma .nf { color: #990000; font-weight: bold }
/* NameFunctionMagic */ .chroma .fm { }
/* NameLabel */ .chroma .nl { color: #990000; font-weight: bold }
/* NameNamespace */ .chroma .nn { color: #555555 }
/* NameOther */ .chroma .nx { }
/* NameProperty */ .chroma .py { }
/* NameTag */ .chroma .nt { color: #000080 }
/* NameVariable */ .chroma .nv { color: #008080 }
/* NameVariableClass */ .chroma .vc { color: #008080 }
/* NameVariableGlobal */ .chroma .vg { color: #008080 }
/* NameVariableInstance */ .chroma .vi { color: #008080 }
/* NameVariableMagic */ .chroma .vm { }
/* Literal */ .chroma .l { }
/* LiteralDate */ .chroma .ld { }
/* LiteralString */ .chroma .s { color: #dd1144 }
/* LiteralStringAffix */ .chroma .sa { color: #dd1144 }
/* LiteralStringBacktick */ .chroma .sb { color: #dd1144 }
/* LiteralStringChar */ .chroma .sc { color: #dd1144 }
/* LiteralStringDelimiter */ .chroma .dl { color: #dd1144 }
/* LiteralStringDoc */ .chroma .sd { color: #dd1144 }
/* LiteralStringDouble */ .chroma .s2 { color: #dd1144 }
/* LiteralStringEscape */ .chroma .se { color: #dd1144 }
/* LiteralStringHeredoc */ .chroma .sh { color: #dd1144 }
/* LiteralStringInterpol */ .chroma .si { color: #dd1144 }
/* LiteralStringOther */ .chroma .sx { color: #dd1144 }
/* LiteralStringRegex */ .chroma .sr { color: #009926 }
/* LiteralStringSingle */ .chroma .s1 { color: #dd1144 }
/* LiteralStringSymbol */ .chroma .ss { color: #990073 }
/* LiteralNumber */ .chroma .m { color: #009999 }
/* LiteralNumberBin */ .chroma .mb { color: #009999 }
/* LiteralNumberFloat */ .chroma .mf { color: #009999 }
/* LiteralNumberHex */ .chroma .mh { color: #009999 }
/* LiteralNumberInteger */ .chroma .mi { color: #009999 }
/* LiteralNumberIntegerLong */ .chroma .il { color: #009999 }
/* LiteralNumberOct */ .chroma .mo { color: #009999 }
/* Operator */ .chroma .o { color: #000000; font-weight: bold }
/* OperatorWord */ .chroma .ow { color: #000000; font-weight: bold }
/* Punctuation */ .chroma .p { }
/* Comment */ .chroma .c { color: #999988; font-style: italic }
/* CommentHashbang */ .chroma .ch { color: #999988; font-style: italic }
/* CommentMultiline */ .chroma .cm { color: #999988; font-style: italic }
/* CommentSingle */ .chroma .c1 { color: #999988; font-style: italic }
/* CommentSpecial */ .chroma .cs { color: #999999; font-weight: bold; font-style: italic }
/* CommentPreproc */ .chroma .cp { color: #999999; font-weight: bold; font-style: italic }
/* CommentPreprocFile */ .chroma .cpf { color: #999999; font-weight: bold; font-style: italic }
/* Generic */ .chroma .g { }
/* GenericDeleted */ .chroma .gd { color: #000000; background-color: #ffdddd }
/* GenericEmph */ .chroma .ge { color: #000000; font-style: italic }
/* GenericError */ .chroma .gr { color: #aa0000 }
/* GenericHeading */ .chroma .gh { color: #999999 }
/* GenericInserted */ .chroma .gi { color: #000000; background-color: #ddffdd }
/* GenericOutput */ .chroma .go { color: #888888 }
/* GenericPrompt */ .chroma .gp { color: #555555 }
/* GenericStrong */ .chroma .gs { font-weight: bold }
/* GenericSubheading */ .chroma .gu { color: #75715e }
/* GenericSubheading */ .chroma .gu { color: #aaaaaa }
/* GenericTraceback */ .chroma .gt { color: #aa0000 }
/* GenericUnderline */ .chroma .gl { text-decoration: underline }
/* TextWhitespace */ .chroma .w { color: #bbbbbb }

1
docs/src/config.toml
View File

@ -5,6 +5,7 @@ theme = "mitmproxydocs"
publishDir = "../public"
RelativeURLs = true
pygmentsCodefences = true
pygmentsUseClasses = true
[indexes]
tag = "tags"

15
docs/src/content/addons-events.md → docs/src/content/addons-api.md
View File

@ -1,8 +1,12 @@
---
title: "Event Hooks"
title: "Event Hooks & API"
url: "api/events.html"
aliases:
- /addons-events/
layout: single
menu:
addons:
weight: 2
weight: 3
---
# Event Hooks
@ -16,9 +20,8 @@ header with a count of the number of responses seen:
{{< example src="examples/addons/http-add-header.py" lang="py" >}}
## Supported Events
## Available Hooks
Below we list events supported by mitmproxy. We've added
annotations to illustrate the argument types.
The following addons list all available event hooks.
{{< readfile file="/generated/events.html" markdown="true" >}}
{{< readfile file="/generated/api/events.html" >}}

4
docs/src/content/addons-examples.md
View File

@ -1,11 +1,9 @@
---
title: "Example Addons"
title: "Examples"
menu:
addons:
weight: 6
---
# Example Addons
{{< readfile file="/generated/examples.html" markdown="true" >}}

61
docs/src/content/addons-overview.md
View File

@ -7,39 +7,19 @@ menu:
# Addons
Mitmproxy's addon mechanism consists of a set of APIs that support components of
any complexity. Addons interact with mitmproxy by responding to **events**,
which allow them to hook into and change mitmproxy's behaviour. They are
configured through **[options]({{< relref concepts-options >}})**, which can be
set in mitmproxy's config file, changed interactively by users, or passed on the
command-line. Finally, they can expose **commands**, which allows users to
invoke their actions either directly or by binding them to keys in the
interactive tools.
Mitmproxy's addon mechanism is an exceptionally powerful part of mitmproxy. In fact, much of mitmproxy's own
functionality is defined in
[a suite of built-in addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons),
implementing everything from functionality like
[anticaching]({{< relref "overview-features#anticache" >}}) and [sticky cookies]({{< relref
"overview-features#sticky-cookies" >}}) to our onboarding webapp.
Addons are an exceptionally powerful part of mitmproxy. In fact, much of
mitmproxy's own functionality is defined in [a suite of built-in
addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons),
implementing everything from functionality like [anticaching]({{< relref
"overview-features#anticache" >}}) and [sticky cookies]({{< relref
"overview-features#sticky-cookies" >}}) to our onboarding webapp. The built-in
addons make for instructive reading, and you will quickly see that quite complex
functionality can often boil down to a very small, completely self-contained
modules. Mitmproxy provides the exact same set of facilities it uses for its own
functionality to third-party scripters and extenders.
Addons interact with mitmproxy by responding to [events]({{< relref addons-api >}}), which allow them to hook into and
change mitmproxy's behaviour. They are configured through [options]({{< relref addons-options >}}), which can be set in
mitmproxy's config file, changed interactively by users, or passed on the command-line. Finally, they can expose
[commands]({{< relref addons-commands >}}), which allows users to invoke their actions either directly or by binding
them to keys in the interactive tools.
This document will show you how to build addons using **events**, **options**
and **commands**. However, this is not an API manual, and the mitmproxy source
code remains the canonical reference. One easy way to explore the API from the
command-line is to use [pydoc](https://docs.python.org/3/library/pydoc.html).
Here, for example, is a command that shows the API documentation for the
mitmproxy's HTTP flow classes:
```bash
pydoc mitmproxy.http
```
You will be referring to the mitmproxy API documentation frequently, so keep
**pydoc** or an equivalent handy.
# Anatomy of an addon
@ -55,7 +35,7 @@ it into your mitmproxy tool of choice. We'll use mitmpdump in these examples,
but the flag is identical for all tools:
```bash
> mitmdump -s ./anatomy.py
mitmdump -s ./anatomy.py
```
Here are a few things to note about the code above:
@ -63,12 +43,21 @@ Here are a few things to note about the code above:
- Mitmproxy picks up the contents of the `addons` global list and loads what it
finds into the addons mechanism.
- Addons are just objects - in this case our addon is an instance of `Counter`.
- The `request` method is an example of an **event**. Addons simply implement a
method for each event they want to handle. Each event has a signature
consisting of arguments that are passed to the method. For `request`, this is
an instance of `mitmproxy.http.HTTPFlow`.
- The `request` method is an example of an *event*. Addons simply implement a
method for each event they want to handle. Each event and its signature are documented
in the [API documentation]({{< relref "addons-api" >}}).
- Finally, the `ctx` module is a holdall module that exposes a set of standard
objects that are commonly used in addons. We could pass a `ctx` object as the
first parameter to every event, but we've found it neater to just expose it as
an importable global. In this case, we're using the `ctx.log` object to do our
logging.
# Abbreviated Scripting Syntax
Sometimes, we would like to write a quick script without going through the trouble of creating a class.
The addons mechanism has a shorthand that allows a module as a whole to be treated as an addon object.
This lets us place event handler functions in the module scope.
For instance, here is a complete script that adds a header to every request:
{{< example src="examples/addons/anatomy2.py" lang="py" >}}

51
docs/src/content/addons-scripting.md
View File

@ -1,51 +0,0 @@
---
title: "Scripting"
menu:
addons:
weight: 5
---
# Scripting HTTP/1.1 and HTTP/2.0
Sometimes, we would like to write a quick script without going through the
trouble of creating a class. The addons mechanism has a shorthand that allows a
module as a whole to be treated as an addon object. This lets us place event
handler functions in the module scope. For instance, here is a complete script
that adds a header to every request.
{{< example src="examples/addons/scripting-minimal-example.py" lang="py" >}}
Here's another example that intercepts requests to a particular URL and sends
an arbitrary response instead:
{{< example src="examples/addons/http-reply-from-proxy.py" lang="py" >}}
All events around the HTTP protocol [can be found here]({{< relref "addons-events#http-events">}}).
For HTTP-related objects, please look at the [http][] module, or the
[Request][], and [Response][] classes for other attributes that you can use when
scripting.
# Scripting WebSocket
The WebSocket protocol initially looks like a regular HTTP request, before the client and server agree to upgrade the connection to WebSocket. All scripting events for initial HTTP handshake, and also the dedicated WebSocket events [can be found here]({{< relref "addons-events#websocket-events">}}).
{{< example src="examples/addons/websocket-simple.py" lang="py" >}}
For WebSocket-related objects please look at the [websocket][] module to find
all attributes that you can use when scripting.
[websocket]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/websocket.py
# Scripting TCP
All events around the TCP protocol [can be found here]({{< relref "addons-events#tcp-events">}}).
{{< example src="examples/addons/tcp-simple.py" lang="py" >}}
For WebSocket-related objects please look at the [tcp][] module to find
all attributes that you can use when scripting.
[tcp]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/tcp.py

11
docs/src/content/api/mitmproxy.addonmanager.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.addonmanager"
url: "api/mitmproxy/addonmanager.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/addonmanager.html" >}}

11
docs/src/content/api/mitmproxy.certs.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.certs"
url: "api/mitmproxy/certs.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/certs.html" >}}

11
docs/src/content/api/mitmproxy.connection.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.connection"
url: "api/mitmproxy/connection.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/connection.html" >}}

11
docs/src/content/api/mitmproxy.coretypes.multidict.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.coretypes.multidict"
url: "api/mitmproxy/coretypes/multidict.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/coretypes/multidict.html" >}}

11
docs/src/content/api/mitmproxy.flow.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.flow"
url: "api/mitmproxy/flow.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/flow.html" >}}

11
docs/src/content/api/mitmproxy.http.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.http"
url: "api/mitmproxy/http.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/http.html" >}}

11
docs/src/content/api/mitmproxy.net.server_spec.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.net.server_spec"
url: "api/mitmproxy/net/server_spec.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/net/server_spec.html" >}}

11
docs/src/content/api/mitmproxy.proxy.server_hooks.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.proxy.server_hooks"
url: "api/mitmproxy/proxy/server_hooks.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/proxy/server_hooks.html" >}}

11
docs/src/content/api/mitmproxy.tcp.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.tcp"
url: "api/mitmproxy/tcp.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/tcp.html" >}}

11
docs/src/content/api/mitmproxy.websocket.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.websocket"
url: "api/mitmproxy/websocket.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/websocket.html" >}}

3
docs/src/layouts/partials/edit-on-github.html
View File

@ -1,4 +1,4 @@
{{ if and .IsPage (not (getenv "DOCS_ARCHIVE")) }}
{{ if and .IsPage (ne .Type "api") (not (getenv "DOCS_ARCHIVE")) }}
<a class="button is-small is-outlined is-link is-pulled-right"
target="_blank"
href="https://github.com/mitmproxy/mitmproxy/blob/master/docs/src/content/{{ .File.Path }}"
@ -6,4 +6,3 @@
Edit on GitHub
</a>
{{ end }}

16
docs/src/layouts/partials/sidemenu.html
View File

@ -3,9 +3,19 @@
{{ $currentPage := .ctx }}
{{ $menuname := .menuname }}
{{ range $menu.ByWeight }}
<li >
<li>
<a class="{{ if $currentPage.IsMenuCurrent $menuname . }}is-active{{ end }}"
href="{{.URL}}">{{ .Name }}</a>
href="{{.URL}}">{{ .Name }}</a>
{{ if and .HasChildren (or ($currentPage.IsMenuCurrent $menuname .) ($currentPage.HasMenuCurrent $menuname .)) }}
<ul>
{{ range .Children }}
<li>
<a class="{{ if $currentPage.IsMenuCurrent $menuname . }}is-active{{ end }}"
href="{{ .URL }}">{{ .Name }}</a>
</li>
{{ end }}
</ul>
{{ end }}
</li>
{{end}}
</ul>
</ul>

3
examples/addons/scripting-minimal-example.py → examples/addons/anatomy2.py
View File

@ -1,2 +1,5 @@
"""An addon using the abbreviated scripting syntax."""
def request(flow):
flow.request.headers["myheader"] = "value"

1
examples/addons/http-trailers.py
View File

@ -36,7 +36,6 @@ def request(flow: http.HTTPFlow):
def response(flow: http.HTTPFlow):
assert flow.response # make type checker happy
if flow.response.trailers:
print("HTTP Trailers detected! Response contains:", flow.response.trailers)

14
examples/addons/internet-in-mirror.py Normal file
View File

@ -0,0 +1,14 @@
"""
Mirror all web pages.
Useful if you are living down under.
"""
from mitmproxy import http
def response(flow: http.HTTPFlow) -> None:
if flow.response and flow.response.content:
flow.response.content = flow.response.content.replace(
b"</head>",
b"<style>body {transform: scaleX(-1);}</style></head>"
)

12
examples/addons/internet_in_mirror.py
View File

@ -1,12 +0,0 @@
"""
Mirror all web pages.
Useful if you are living down under.
"""
from mitmproxy import http
def response(flow: http.HTTPFlow) -> None:
assert flow.response # make type checker happy
reflector = b"<style>body {transform: scaleX(-1);}</style></head>"
flow.response.content = flow.response.content.replace(b"</head>", reflector)

24
examples/addons/websocket-inject-message.py
View File

@ -1,24 +0,0 @@
"""
Inject a WebSocket message into a running connection.
This example shows how to inject a WebSocket message to the client.
Every new WebSocket connection will trigger a new asyncio task that
periodically injects a new message to the client.
"""
import asyncio
import mitmproxy.websocket
class InjectWebSocketMessage:
async def inject(self, flow: mitmproxy.websocket.WebSocketFlow):
i = 0
while not flow.ended and not flow.error:
await asyncio.sleep(5)
flow.inject_message(flow.client_conn, f'This is the #{i} injected message!')
i += 1
def websocket_start(self, flow):
asyncio.get_event_loop().create_task(self.inject(flow))
addons = [InjectWebSocketMessage()]

5
mitmproxy/addonmanager.py
View File

@ -98,6 +98,11 @@ class Loader:
)
def add_command(self, path: str, func: typing.Callable) -> None:
"""Add a command to mitmproxy.
Unless you are generating commands programatically,
this API should be avoided. Decorate your function with `@mitmproxy.command.command` instead.
"""
self.master.commands.add(path, func)

8
mitmproxy/addons/dumper.py
View File

@ -80,11 +80,11 @@ class Dumper:
def _echo_headers(self, headers: http.Headers):
for k, v in headers.fields:
k = strutils.bytes_to_escaped_str(k)
v = strutils.bytes_to_escaped_str(v)
ks = strutils.bytes_to_escaped_str(k)
vs = strutils.bytes_to_escaped_str(v)
out = "{}: {}".format(
click.style(k, fg="blue"),
click.style(v)
click.style(ks, fg="blue"),
click.style(vs)
)
self.echo(out, ident=4)

1
mitmproxy/certs.py
View File

@ -39,6 +39,7 @@ rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI=
class Cert(serializable.Serializable):
"""Representation of a (TLS) certificate."""
_cert: x509.Certificate
def __init__(self, cert: x509.Certificate):

8
mitmproxy/connection.py
View File

@ -29,7 +29,7 @@ class Connection(serializable.Serializable, metaclass=ABCMeta):
Base class for client and server connections.
The connection object only exposes metadata about the connection, but not the underlying socket object.
This is intentional, all I/O should be handled by mitmproxy.proxy.server exclusively.
This is intentional, all I/O should be handled by `mitmproxy.proxy.server` exclusively.
"""
# all connections have a unique id. While
# f.client_conn == f2.client_conn already holds true for live flows (where we have object identity),
@ -92,12 +92,12 @@ class Connection(serializable.Serializable, metaclass=ABCMeta):
@property
def connected(self) -> bool:
"""`True` if Connection.state is ConnectionState.OPEN, `False` otherwise. Read-only."""
"""*Read-only:* `True` if Connection.state is ConnectionState.OPEN, `False` otherwise."""
return self.state is ConnectionState.OPEN
@property
def tls_established(self) -> bool:
"""`True` if TLS has been established, `False` otherwise. Read-only."""
"""*Read-only:* `True` if TLS has been established, `False` otherwise."""
return self.timestamp_tls_setup is not None
def __eq__(self, other):
@ -143,7 +143,7 @@ class Client(Connection):
timestamp_start: float
"""*Timestamp:* TCP SYN received"""
def __init__(self, peername, sockname, timestamp_start):
def __init__(self, peername: Address, sockname: Address, timestamp_start: float):
self.id = str(uuid.uuid4())
self.peername = peername
self.sockname = sockname

83
mitmproxy/coretypes/multidict.py
View File

@ -1,10 +1,26 @@
from abc import ABCMeta, abstractmethod
from abc import ABCMeta
from abc import abstractmethod
from typing import Iterator
from typing import List
from typing import MutableMapping
from typing import Sequence
from typing import Tuple
from typing import TypeVar
from collections.abc import MutableMapping
from mitmproxy.coretypes import serializable
KT = TypeVar('KT')
VT = TypeVar('VT')
class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta):
"""
A MultiDict is a dictionary-like data structure that supports multiple values per key.
"""
fields: Tuple[Tuple[KT, VT], ...]
"""The underlying raw datastructure."""
class _MultiDict(MutableMapping, metaclass=ABCMeta):
def __repr__(self):
fields = (
repr(field)
@ -17,7 +33,7 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
@staticmethod
@abstractmethod
def _reduce_values(values):
def _reduce_values(values: Sequence[VT]) -> VT:
"""
If a user accesses multidict["foo"], this method
reduces all values for "foo" to a single value that is returned.
@ -27,22 +43,22 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
@staticmethod
@abstractmethod
def _kconv(key):
def _kconv(key: KT) -> KT:
"""
This method converts a key to its canonical representation.
For example, HTTP headers are case-insensitive, so this method returns key.lower().
"""
def __getitem__(self, key):
def __getitem__(self, key: KT) -> VT:
values = self.get_all(key)
if not values:
raise KeyError(key)
return self._reduce_values(values)
def __setitem__(self, key, value):
def __setitem__(self, key: KT, value: VT) -> None:
self.set_all(key, [value])
def __delitem__(self, key):
def __delitem__(self, key: KT) -> None:
if key not in self:
raise KeyError(key)
key = self._kconv(key)
@ -51,7 +67,7 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
if key != self._kconv(field[0])
)
def __iter__(self):
def __iter__(self) -> Iterator[KT]:
seen = set()
for key, _ in self.fields:
key_kconv = self._kconv(key)
@ -59,15 +75,15 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
seen.add(key_kconv)
yield key
def __len__(self):
def __len__(self) -> int:
return len({self._kconv(key) for key, _ in self.fields})
def __eq__(self, other):
def __eq__(self, other) -> bool:
if isinstance(other, MultiDict):
return self.fields == other.fields
return False
def get_all(self, key):
def get_all(self, key: KT) -> List[VT]:
"""
Return the list of all values for a given key.
If that key is not in the MultiDict, the return value will be an empty list.
@ -79,13 +95,13 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
if self._kconv(k) == key
]
def set_all(self, key, values):
def set_all(self, key: KT, values: List[VT]) -> None:
"""
Remove the old values for a key and add new ones.
"""
key_kconv = self._kconv(key)
new_fields = []
new_fields: List[Tuple[KT, VT]] = []
for field in self.fields:
if self._kconv(field[0]) == key_kconv:
if values:
@ -100,55 +116,49 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
)
self.fields = tuple(new_fields)
def add(self, key, value):
def add(self, key: KT, value: VT) -> None:
"""
Add an additional value for the given key at the bottom.
"""
self.insert(len(self.fields), key, value)
def insert(self, index, key, value):
def insert(self, index: int, key: KT, value: VT) -> None:
"""
Insert an additional value for the given key at the specified position.
"""
item = (key, value)
self.fields = self.fields[:index] + (item,) + self.fields[index:]
def keys(self, multi=False):
def keys(self, multi: bool = False):
"""
Get all keys.
Args:
multi(bool):
If True, one key per value will be returned.
If False, duplicate keys will only be returned once.
If `multi` is True, one key per value will be returned.
If `multi` is False, duplicate keys will only be returned once.
"""
return (
k
for k, _ in self.items(multi)
)
def values(self, multi=False):
def values(self, multi: bool = False):
"""
Get all values.
Args:
multi(bool):
If True, all values will be returned.
If False, only the first value per key will be returned.
If `multi` is True, all values will be returned.
If `multi` is False, only the first value per key will be returned.
"""
return (
v
for _, v in self.items(multi)
)
def items(self, multi=False):
def items(self, multi: bool = False):
"""
Get all (key, value) tuples.
Args:
multi(bool):
If True, all (key, value) pairs will be returned
If False, only the first (key, value) pair per unique key will be returned.
If `multi` is True, all `(key, value)` pairs will be returned.
If False, only one tuple per key is returned.
"""
if multi:
return self.fields
@ -156,7 +166,9 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
return super().items()
class MultiDict(_MultiDict, serializable.Serializable):
class MultiDict(_MultiDict[KT, VT], serializable.Serializable):
"""A concrete MultiDict, storing its own data."""
def __init__(self, fields=()):
super().__init__()
self.fields = tuple(
@ -182,12 +194,13 @@ class MultiDict(_MultiDict, serializable.Serializable):
return cls(state)
class MultiDictView(_MultiDict):
class MultiDictView(_MultiDict[KT, VT]):
"""
The MultiDictView provides the MultiDict interface over calculated data.
The view itself contains no state - data is retrieved from the parent on
request, and stored back to the parent on change.
"""
def __init__(self, getter, setter):
self._getter = getter
self._setter = setter
@ -204,7 +217,7 @@ class MultiDictView(_MultiDict):
# multiple elements exist with the same key.
return values[0]
@property
@property # type: ignore
def fields(self):
return self._getter()
@ -212,5 +225,5 @@ class MultiDictView(_MultiDict):
def fields(self, value):
self._setter(value)
def copy(self):
def copy(self) -> "MultiDict[KT,VT]":
return MultiDict(self.fields)

84
mitmproxy/flow.py
View File

@ -10,21 +10,21 @@ from mitmproxy import version
class Error(stateobject.StateObject):
"""
An Error.
An Error.
This is distinct from an protocol error response (say, a HTTP code 500),
which is represented by a normal `mitmproxy.http.Response` object. This class is
responsible for indicating errors that fall outside of normal protocol
communications, like interrupted connections, timeouts, protocol errors.
This is distinct from an protocol error response (say, a HTTP code 500),
which is represented by a normal `mitmproxy.http.Response` object. This class is
responsible for indicating errors that fall outside of normal protocol
communications, like interrupted connections, timeouts, or protocol errors.
"""
msg: str
"""Message describing the error."""
timestamp: float
"""Unix timestamp"""
"""Unix timestamp of when this error happened."""
KILLED_MESSAGE = "Connection killed."
KILLED_MESSAGE: typing.ClassVar[str] = "Connection killed."
def __init__(self, msg: str, timestamp: typing.Optional[float] = None) -> None:
"""Create an error. If no timestamp is passed, the current time is used."""
@ -53,8 +53,46 @@ class Error(stateobject.StateObject):
class Flow(stateobject.StateObject):
"""
A Flow is a collection of objects representing a single transaction.
This class is usually subclassed for each protocol, e.g. HTTPFlow.
Base class for network flows. A flow is a collection of objects,
for example HTTP request/response pairs or a list of TCP messages.
See also:
- mitmproxy.http.HTTPFlow
- mitmproxy.tcp.TCPFlow
"""
client_conn: connection.Client
"""The client that connected to mitmproxy."""
server_conn: connection.Server
"""
The server mitmproxy connected to.
Some flows may never cause mitmproxy to initiate a server connection,
for example because their response is replayed by mitmproxy itself.
To simplify implementation, those flows will still have a `server_conn` attribute
with a `timestamp_start` set to `None`.
"""
error: typing.Optional[Error] = None
"""A connection or protocol error affecting this flow."""
intercepted: bool
"""
If `True`, the flow is currently paused by mitmproxy.
We're waiting for a user action to forward the flow to its destination.
"""
marked: bool
"""
If `True`, this flow has been marked by the user.
"""
is_replay: typing.Optional[str]
"""
This attribute indicates if this flow has been replayed in either direction.
- a value of `request` indicates that the request has been artifically replayed by mitmproxy to the server.
- a value of `response` indicates that the response to the client's request has been set by server replay.
"""
def __init__(
@ -70,7 +108,6 @@ class Flow(stateobject.StateObject):
self.server_conn = server_conn
self.live = live
self.error: typing.Optional[Error] = None
self.intercepted: bool = False
self._backup: typing.Optional[Flow] = None
self.reply: typing.Optional[controller.Reply] = None
@ -111,6 +148,7 @@ class Flow(stateobject.StateObject):
return f
def copy(self):
"""Make a copy of this flow."""
f = super().copy()
f.live = False
if self.reply is not None:
@ -119,7 +157,7 @@ class Flow(stateobject.StateObject):
def modified(self):
"""
Has this Flow been modified?
`True` if this file has been modified by a user, `False` otherwise.
"""
if self._backup:
return self._backup != self.get_state()
@ -128,8 +166,7 @@ class Flow(stateobject.StateObject):
def backup(self, force=False):
"""
Save a backup of this Flow, which can be reverted to using a
call to .revert().
Save a backup of this flow, which can be restored by calling `Flow.revert()`.
"""
if not self._backup:
self._backup = self.get_state()
@ -144,6 +181,7 @@ class Flow(stateobject.StateObject):
@property
def killable(self):
"""*Read-only:* `True` if this flow can be killed, `False` otherwise."""
return (
self.reply and
self.reply.state in {"start", "taken"} and
@ -152,7 +190,7 @@ class Flow(stateobject.StateObject):
def kill(self):
"""
Kill this request.
Kill this flow. The current request/response will not be forwarded to its destination.
"""
if not self.killable:
raise exceptions.ControlException("Flow is not killable.")
@ -162,8 +200,8 @@ class Flow(stateobject.StateObject):
def intercept(self):
"""
Intercept this Flow. Processing will stop until resume is
called.
Intercept this Flow. Processing will stop until resume is
called.
"""
if self.intercepted:
return
@ -172,7 +210,7 @@ class Flow(stateobject.StateObject):
def resume(self):
"""
Continue with the flow - called after an intercept().
Continue with the flow – called after an intercept().
"""
if not self.intercepted:
return
@ -183,5 +221,15 @@ class Flow(stateobject.StateObject):
@property
def timestamp_start(self) -> float:
"""Start time of the flow."""
"""
*Read-only:* Start time of the flow.
Depending on the flow type, this property is an alias for
`mitmproxy.connection.Client.timestamp_start` or `mitmproxy.http.Request.timestamp_start`.
"""
return self.client_conn.timestamp_start
__all__ = [
"Flow",
"Error",
]

438
mitmproxy/http.py
View File

@ -3,23 +3,30 @@ import time
import urllib.parse
from dataclasses import dataclass
from dataclasses import fields
from email.utils import formatdate, mktime_tz, parsedate_tz
from typing import Callable, cast
from email.utils import formatdate
from email.utils import mktime_tz
from email.utils import parsedate_tz
from typing import Callable
from typing import Dict
from typing import Iterable
from typing import Iterator
from typing import List
from typing import Mapping
from typing import Optional
from typing import Tuple
from typing import Union
from typing import cast
from mitmproxy import flow, connection
from mitmproxy import flow
from mitmproxy.coretypes import multidict
from mitmproxy.coretypes import serializable
from mitmproxy.net import encoding
from mitmproxy.net.http import cookies, multipart
from mitmproxy.net.http import cookies
from mitmproxy.net.http import multipart
from mitmproxy.net.http import status_codes
from mitmproxy.net.http import url
from mitmproxy.net.http.headers import assemble_content_type, parse_content_type
from mitmproxy.net.http.headers import assemble_content_type
from mitmproxy.net.http.headers import parse_content_type
from mitmproxy.utils import human
from mitmproxy.utils import strutils
from mitmproxy.utils import typecheck
@ -28,72 +35,70 @@ from mitmproxy.utils.strutils import always_str
# While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
def _native(x):
def _native(x: bytes) -> str:
return x.decode("utf-8", "surrogateescape")
def _always_bytes(x):
def _always_bytes(x: Union[str, bytes]) -> bytes:
return strutils.always_bytes(x, "utf-8", "surrogateescape")
class Headers(multidict.MultiDict):
# This cannot be easily typed with mypy yet, so we just specify MultiDict without concrete types.
class Headers(multidict.MultiDict): # type: ignore
"""
Header class which allows both convenient access to individual headers as well as
direct access to the underlying raw data. Provides a full dictionary interface.
Example:
Create headers with keyword arguments:
>>> h = Headers(host="example.com", content_type="application/xml")
.. code-block:: python
Headers mostly behave like a normal dict:
>>> h["Host"]
"example.com"
# Create headers with keyword arguments
>>> h = Headers(host="example.com", content_type="application/xml")
Headers are case insensitive:
>>> h["host"]
"example.com"
# Headers mostly behave like a normal dict.
>>> h["Host"]
"example.com"
Headers can also be created from a list of raw (header_name, header_value) byte tuples:
>>> h = Headers([
(b"Host",b"example.com"),
(b"Accept",b"text/html"),
(b"accept",b"application/xml")
])
# HTTP Headers are case insensitive
>>> h["host"]
"example.com"
Multiple headers are folded into a single header as per RFC 7230:
>>> h["Accept"]
"text/html, application/xml"
# Headers can also be created from a list of raw (header_name, header_value) byte tuples
>>> h = Headers([
(b"Host",b"example.com"),
(b"Accept",b"text/html"),
(b"accept",b"application/xml")
])
Setting a header removes all existing headers with the same name:
>>> h["Accept"] = "application/text"
>>> h["Accept"]
"application/text"
# Multiple headers are folded into a single header as per RFC7230
>>> h["Accept"]
"text/html, application/xml"
`bytes(h)` returns an HTTP/1 header block:
>>> print(bytes(h))
Host: example.com
Accept: application/text
# Setting a header removes all existing headers with the same name.
>>> h["Accept"] = "application/text"
>>> h["Accept"]
"application/text"
# bytes(h) returns a HTTP1 header block.
>>> print(bytes(h))
Host: example.com
Accept: application/text
# For full control, the raw header fields can be accessed
>>> h.fields
For full control, the raw header fields can be accessed:
>>> h.fields
Caveats:
For use with the "Set-Cookie" header, see :py:meth:`get_all`.
- For use with the "Set-Cookie" header, either use `Response.cookies` or see `Headers.get_all`.
"""
def __init__(self, fields=(), **headers):
def __init__(self, fields: Iterable[Tuple[bytes, bytes]] = (), **headers):
"""
Args:
fields: (optional) list of ``(name, value)`` header byte tuples,
e.g. ``[(b"Host", b"example.com")]``. All names and values must be bytes.
**headers: Additional headers to set. Will overwrite existing values from `fields`.
For convenience, underscores in header names will be transformed to dashes -
this behaviour does not extend to other methods.
If ``**headers`` contains multiple keys that have equal ``.lower()`` s,
the behavior is undefined.
*Args:*
- *fields:* (optional) list of ``(name, value)`` header byte tuples,
e.g. ``[(b"Host", b"example.com")]``. All names and values must be bytes.
- *\*\*headers:* Additional headers to set. Will overwrite existing values from `fields`.
For convenience, underscores in header names will be transformed to dashes -
this behaviour does not extend to other methods.
If ``**headers`` contains multiple keys that have equal ``.lower()`` representations,
the behavior is undefined.
"""
super().__init__(fields)
@ -102,41 +107,43 @@ class Headers(multidict.MultiDict):
raise TypeError("Header fields must be bytes.")
# content_type -> content-type
headers = {
self.update({
_always_bytes(name).replace(b"_", b"-"): _always_bytes(value)
for name, value in headers.items()
}
self.update(headers)
})
fields: Tuple[Tuple[bytes, bytes], ...]
@staticmethod
def _reduce_values(values):
def _reduce_values(values) -> str:
# Headers can be folded
return ", ".join(values)
@staticmethod
def _kconv(key):
def _kconv(key) -> str:
# Headers are case-insensitive
return key.lower()
def __bytes__(self):
def __bytes__(self) -> bytes:
if self.fields:
return b"\r\n".join(b": ".join(field) for field in self.fields) + b"\r\n"
else:
return b""
def __delitem__(self, key):
def __delitem__(self, key: Union[str, bytes]) -> None:
key = _always_bytes(key)
super().__delitem__(key)
def __iter__(self):
def __iter__(self) -> Iterator[str]:
for x in super().__iter__():
yield _native(x)
def get_all(self, name):
def get_all(self, name: Union[str, bytes]) -> List[str]:
"""
Like :py:meth:`get`, but does not fold multiple headers into a single one.
Like `Headers.get`, but does not fold multiple headers into a single one.
This is useful for Set-Cookie headers, which do not support folding.
See also: https://tools.ietf.org/html/rfc7230#section-3.2.2
*See also:* <https://tools.ietf.org/html/rfc7230#section-3.2.2>
"""
name = _always_bytes(name)
return [
@ -144,16 +151,16 @@ class Headers(multidict.MultiDict):
super().get_all(name)
]
def set_all(self, name, values):
def set_all(self, name: Union[str, bytes], values: List[Union[str, bytes]]):
"""
Explicitly set multiple headers for the given key.
See: :py:meth:`get_all`
See `Headers.get_all`.
"""
name = _always_bytes(name)
values = [_always_bytes(x) for x in values]
return super().set_all(name, values)
def insert(self, index, key, value):
def insert(self, index: int, key: Union[str, bytes], value: Union[str, bytes]):
key = _always_bytes(key)
value = _always_bytes(value)
super().insert(index, key, value)
@ -222,6 +229,8 @@ class ResponseData(MessageData):
class Message(serializable.Serializable):
"""Base class for `Request` and `Response`."""
@classmethod
def from_state(cls, state):
return cls(**state)
@ -233,12 +242,21 @@ class Message(serializable.Serializable):
self.data.set_state(state)
data: MessageData
stream: Union[Callable, bool] = False
stream: Union[Callable[[bytes], bytes], bool] = False
"""
If `True`, the message body will not be buffered on the proxy
but immediately streamed to the destination instead.
Alternatively, a transformation function can be specified, but please note
that packet should not be relied upon.
This attribute must be set in the `requestheaders` or `responseheaders` hook.
Setting it in `request` or `response` is already too late, mitmproxy has buffered the message body already.
"""
@property
def http_version(self) -> str:
"""
Version string, e.g. "HTTP/1.1"
HTTP version string, for example `HTTP/1.1`.
"""
return self.data.http_version.decode("utf-8", "surrogateescape")
@ -272,7 +290,7 @@ class Message(serializable.Serializable):
@property
def trailers(self) -> Optional[Headers]:
"""
The HTTP trailers.
The [HTTP trailers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Trailer).
"""
return self.data.trailers
@ -283,9 +301,11 @@ class Message(serializable.Serializable):
@property
def raw_content(self) -> Optional[bytes]:
"""
The raw (potentially compressed) HTTP message body as bytes.
The raw (potentially compressed) HTTP message body.
See also: :py:attr:`content`, :py:class:`text`
In contrast to `Message.content` and `Message.text`, accessing this property never raises.
*See also:* `Message.content`, `Message.text`
"""
return self.data.content
@ -293,31 +313,35 @@ class Message(serializable.Serializable):
def raw_content(self, content: Optional[bytes]) -> None:
self.data.content = content
def get_content(self, strict: bool = True) -> Optional[bytes]:
@property
def content(self) -> Optional[bytes]:
"""
The uncompressed HTTP message body as bytes.
Raises:
ValueError, when the HTTP content-encoding is invalid and strict is True.
Accessing this attribute may raise a `ValueError` when the HTTP content-encoding is invalid.
See also: :py:class:`raw_content`, :py:attr:`text`
*See also:* `Message.raw_content`, `Message.text`
"""
if self.raw_content is None:
return None
ce = self.headers.get("content-encoding")
if ce:
try:
content = encoding.decode(self.raw_content, ce)
# A client may illegally specify a byte -> str encoding here (e.g. utf8)
if isinstance(content, str):
raise ValueError(f"Invalid Content-Encoding: {ce}")
return content
except ValueError:
if strict:
raise
return self.raw_content
else:
return self.raw_content
return self.get_content()
@content.setter
def content(self, value: Optional[bytes]) -> None:
self.set_content(value)
@property
def text(self) -> Optional[str]:
"""
The uncompressed and decoded HTTP message body as text.
Accessing this attribute may raise a `ValueError` when either content-encoding or charset is invalid.
*See also:* `Message.raw_content`, `Message.content`
"""
return self.get_text()
@text.setter
def text(self, value: Optional[str]) -> None:
self.set_text(value)
def set_content(self, value: Optional[bytes]) -> None:
if value is None:
@ -338,29 +362,27 @@ class Message(serializable.Serializable):
self.raw_content = value
self.headers["content-length"] = str(len(self.raw_content))
content = property(get_content, set_content)
@property
def timestamp_start(self) -> float:
def get_content(self, strict: bool = True) -> Optional[bytes]:
"""
First byte timestamp
Similar to `Message.content`, but does not raise if `strict` is `False`.
Instead, the compressed message body is returned as-is.
"""
return self.data.timestamp_start
@timestamp_start.setter
def timestamp_start(self, timestamp_start: float) -> None:
self.data.timestamp_start = timestamp_start
@property
def timestamp_end(self) -> Optional[float]:
"""
Last byte timestamp
"""
return self.data.timestamp_end
@timestamp_end.setter
def timestamp_end(self, timestamp_end: Optional[float]):
self.data.timestamp_end = timestamp_end
if self.raw_content is None:
return None
ce = self.headers.get("content-encoding")
if ce:
try:
content = encoding.decode(self.raw_content, ce)
# A client may illegally specify a byte -> str encoding here (e.g. utf8)
if isinstance(content, str):
raise ValueError(f"Invalid Content-Encoding: {ce}")
return content
except ValueError:
if strict:
raise
return self.raw_content
else:
return self.raw_content
def _get_content_type_charset(self) -> Optional[str]:
ct = parse_content_type(self.headers.get("content-type", ""))
@ -391,14 +413,26 @@ class Message(serializable.Serializable):
return enc
def set_text(self, text: Optional[str]) -> None:
if text is None:
self.content = None
return
enc = self._guess_encoding()
try:
self.content = cast(bytes, encoding.encode(text, enc))
except ValueError:
# Fall back to UTF-8 and update the content-type header.
ct = parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {})
ct[2]["charset"] = "utf-8"
self.headers["content-type"] = assemble_content_type(*ct)
enc = "utf8"
self.content = text.encode(enc, "surrogateescape")
def get_text(self, strict: bool = True) -> Optional[str]:
"""
The uncompressed and decoded HTTP message body as text.
Raises:
ValueError, when either content-encoding or charset is invalid and strict is True.
See also: :py:attr:`content`, :py:class:`raw_content`
Similar to `Message.text`, but does not raise if `strict` is `False`.
Instead, the message body is returned as surrogate-escaped UTF-8.
"""
content = self.get_content(strict)
if content is None:
@ -411,23 +445,27 @@ class Message(serializable.Serializable):
raise
return content.decode("utf8", "surrogateescape")
def set_text(self, text: Optional[str]) -> None:
if text is None:
self.content = None
return
enc = self._guess_encoding()
@property
def timestamp_start(self) -> float:
"""
*Timestamp:* Headers received.
"""
return self.data.timestamp_start
try:
self.content = encoding.encode(text, enc)
except ValueError:
# Fall back to UTF-8 and update the content-type header.
ct = parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {})
ct[2]["charset"] = "utf-8"
self.headers["content-type"] = assemble_content_type(*ct)
enc = "utf8"
self.content = text.encode(enc, "surrogateescape")
@timestamp_start.setter
def timestamp_start(self, timestamp_start: float) -> None:
self.data.timestamp_start = timestamp_start
text = property(get_text, set_text)
@property
def timestamp_end(self) -> Optional[float]:
"""
*Timestamp:* Last byte received.
"""
return self.data.timestamp_end
@timestamp_end.setter
def timestamp_end(self, timestamp_end: Optional[float]):
self.data.timestamp_end = timestamp_end
def decode(self, strict: bool = True) -> None:
"""
@ -435,26 +473,25 @@ class Message(serializable.Serializable):
removes the header. If there is no Content-Encoding header, no
action is taken.
Raises:
ValueError, when the content-encoding is invalid and strict is True.
*Raises:*
- `ValueError`, when the content-encoding is invalid and strict is True.
"""
decoded = self.get_content(strict)
self.headers.pop("content-encoding", None)
self.content = decoded
def encode(self, e: str) -> None:
def encode(self, encoding: str) -> None:
"""
Encodes body with the encoding e, where e is "gzip", "deflate", "identity", "br", or "zstd".
Any existing content-encodings are overwritten,
the content is not decoded beforehand.
Encodes body with the given encoding, where e is "gzip", "deflate", "identity", "br", or "zstd".
Any existing content-encodings are overwritten, the content is not decoded beforehand.
Raises:
ValueError, when the specified content-encoding is invalid.
*Raises:*
- `ValueError`, when the specified content-encoding is invalid.
"""
self.headers["content-encoding"] = e
self.headers["content-encoding"] = encoding
self.content = self.raw_content
if "content-encoding" not in self.headers:
raise ValueError("Invalid content encoding {}".format(repr(e)))
raise ValueError("Invalid content encoding {}".format(repr(encoding)))
class Request(Message):
@ -474,7 +511,7 @@ class Request(Message):
http_version: bytes,
headers: Union[Headers, Tuple[Tuple[bytes, bytes], ...]],
content: Optional[bytes],
trailers: Union[None, Headers, Tuple[Tuple[bytes, bytes], ...]],
trailers: Union[Headers, Tuple[Tuple[bytes, bytes], ...], None],
timestamp_start: float,
timestamp_end: Optional[float],
):
@ -543,7 +580,7 @@ class Request(Message):
for k, v in headers.items()
)
elif isinstance(headers, Iterable):
headers = Headers(headers)
headers = Headers(headers) # type: ignore
else:
raise TypeError("Expected headers to be an iterable or dict, but is {}.".format(
type(headers).__name__
@ -578,7 +615,7 @@ class Request(Message):
@property
def first_line_format(self) -> str:
"""
HTTP request form as defined in `RFC7230 <https://tools.ietf.org/html/rfc7230#section-5.3>`_.
*Read-only:* HTTP request form as defined in [RFC 7230](https://tools.ietf.org/html/rfc7230#section-5.3).
origin-form and asterisk-form are subsumed as "relative".
"""
@ -617,9 +654,12 @@ class Request(Message):
HTTP request authority.
For HTTP/1, this is the authority portion of the request target
(in either absolute-form or authority-form)
(in either absolute-form or authority-form).
For origin-form and asterisk-form requests, this property is set to an empty string.
For HTTP/2, this is the :authority pseudo header.
*See also:* `Request.host`, `Request.host_header`, `Request.pretty_host`
"""
try:
return self.data.authority.decode("idna")
@ -638,11 +678,13 @@ class Request(Message):
@property
def host(self) -> str:
"""
Target host. This may be parsed from the raw request
Target server for this request. This may be parsed from the raw request
(e.g. from a ``GET http://example.com/ HTTP/1.1`` request line)
or inferred from the proxy mode (e.g. an IP in transparent mode).
Setting the host attribute also updates the host header and authority information, if present.
*See also:* `Request.authority`, `Request.host_header`, `Request.pretty_host`
"""
return self.data.host
@ -664,6 +706,8 @@ class Request(Message):
This property maps to either ``request.headers["Host"]`` or
``request.authority``, depending on whether it's HTTP/1.x or HTTP/2.0.
*See also:* `Request.authority`,`Request.host`, `Request.pretty_host`
"""
if self.is_http2:
return self.authority or self.data.headers.get("Host", None)
@ -686,7 +730,7 @@ class Request(Message):
@property
def port(self) -> int:
"""
Target port
Target port.
"""
return self.data.port
@ -709,7 +753,9 @@ class Request(Message):
@property
def url(self) -> str:
"""
The URL string, constructed from the request's URL components.
The full URL string, constructed from `Request.scheme`, `Request.host`, `Request.port` and `Request.path`.
Settings this property updates these attributes as well.
"""
if self.first_line_format == "authority":
return f"{self.host}:{self.port}"
@ -723,9 +769,11 @@ class Request(Message):
@property
def pretty_host(self) -> str:
"""
Similar to :py:attr:`host`, but using the host/:authority header as an additional (preferred) data source.
This is useful in transparent mode where :py:attr:`host` is only an IP address,
but may not reflect the actual destination as the Host header could be spoofed.
*Read-only:* Like `Request.host`, but using `Request.host_header` header as an additional (preferred) data source.
This is useful in transparent mode where `Request.host` is only an IP address.
*Warning:* When working in adversarial environments, this may not reflect the actual destination
as the Host header could be spoofed.
"""
authority = self.host_header
if authority:
@ -736,7 +784,7 @@ class Request(Message):
@property
def pretty_url(self) -> str:
"""
Like :py:attr:`url`, but using :py:attr:`pretty_host` instead of :py:attr:`host`.
*Read-only:* Like `Request.url`, but using `Request.pretty_host` instead of `Request.host`.
"""
if self.first_line_format == "authority":
return self.authority
@ -760,9 +808,11 @@ class Request(Message):
self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment])
@property
def query(self) -> multidict.MultiDictView:
def query(self) -> multidict.MultiDictView[str, str]:
"""
The request query string as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
The request query as a mutable mapping view on the request's path.
For the most part, this behaves like a dictionary.
Modifications to the MultiDictView update `Request.path`, and vice versa.
"""
return multidict.MultiDictView(
self._get_query,
@ -781,11 +831,11 @@ class Request(Message):
self.headers["cookie"] = cookies.format_cookie_header(value)
@property
def cookies(self) -> multidict.MultiDictView:
def cookies(self) -> multidict.MultiDictView[str, str]:
"""
The request cookies.
An empty :py:class:`~mitmproxy.net.multidict.MultiDictView` object if the cookie monster ate them all.
For the most part, this behaves like a dictionary.
Modifications to the MultiDictView update `Request.headers`, and vice versa.
"""
return multidict.MultiDictView(
self._get_cookies,
@ -797,7 +847,7 @@ class Request(Message):
self._set_cookies(value)
@property
def path_components(self):
def path_components(self) -> Tuple[str, ...]:
"""
The URL's path components as a tuple of strings.
Components are unquoted.
@ -809,7 +859,7 @@ class Request(Message):
return tuple(url.unquote(i) for i in path.split("/") if i)
@path_components.setter
def path_components(self, components):
def path_components(self, components: Iterable[str]):
components = map(lambda x: url.quote(x, safe=""), components)
path = "/" + "/".join(components)
_, _, _, params, query, fragment = urllib.parse.urlparse(self.url)
@ -817,27 +867,24 @@ class Request(Message):
def anticache(self) -> None:
"""
Modifies this request to remove headers that might produce a cached
response. That is, we remove ETags and If-Modified-Since headers.
Modifies this request to remove headers that might produce a cached response.
"""
delheaders = [
delheaders = (
"if-modified-since",
"if-none-match",
]
)
for i in delheaders:
self.headers.pop(i, None)
def anticomp(self) -> None:
"""
Modifies this request to remove headers that will compress the
resource's data.
Modify the Accept-Encoding header to only accept uncompressed responses.
"""
self.headers["accept-encoding"] = "identity"
def constrain_encoding(self) -> None:
"""
Limits the permissible Accept-Encoding values, based on what we can
decode appropriately.
Limits the permissible Accept-Encoding values, based on what we can decode appropriately.
"""
accept_encoding = self.headers.get("accept-encoding")
if accept_encoding:
@ -864,13 +911,14 @@ class Request(Message):
self.content = url.encode(form_data, self.get_text(strict=False)).encode()
@property
def urlencoded_form(self):
def urlencoded_form(self) -> multidict.MultiDictView[str, str]:
"""
The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
An empty multidict.MultiDictView if the content-type indicates non-form data
or the content could not be parsed.
The URL-encoded form data.
Starting with mitmproxy 1.0, key and value are strings.
If the content-type indicates non-form data or the form could not be parsed, this is set to
an empty `MultiDictView`.
Modifications to the MultiDictView update `Request.content`, and vice versa.
"""
return multidict.MultiDictView(
self._get_urlencoded_form,
@ -895,13 +943,14 @@ class Request(Message):
self.headers["content-type"] = "multipart/form-data"
@property
def multipart_form(self):
def multipart_form(self) -> multidict.MultiDictView[bytes, bytes]:
"""
The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
An empty multidict.MultiDictView if the content-type indicates non-form data
or the content could not be parsed.
The multipart form data.
Key and value are bytes.
If the content-type indicates non-form data or the form could not be parsed, this is set to
an empty `MultiDictView`.
Modifications to the MultiDictView update `Request.content`, and vice versa.
"""
return multidict.MultiDictView(
self._get_multipart_form,
@ -977,12 +1026,12 @@ class Response(Message):
headers = headers
elif isinstance(headers, dict):
headers = Headers(
(always_bytes(k, "utf-8", "surrogateescape"),
(always_bytes(k, "utf-8", "surrogateescape"), # type: ignore
always_bytes(v, "utf-8", "surrogateescape"))
for k, v in headers.items()
)
elif isinstance(headers, Iterable):
headers = Headers(headers)
headers = Headers(headers) # type: ignore
else:
raise TypeError("Expected headers to be an iterable or dict, but is {}.".format(
type(headers).__name__
@ -1023,7 +1072,8 @@ class Response(Message):
@property
def reason(self) -> str:
"""
HTTP Reason Phrase, e.g. "Not Found".
HTTP reason phrase, for example "Not Found".
HTTP/2 responses do not contain a reason phrase, an empty string will be returned instead.
"""
# Encoding: http://stackoverflow.com/a/16674906/934719
@ -1049,16 +1099,15 @@ class Response(Message):
self.headers.set_all("set-cookie", cookie_headers)
@property
def cookies(self) -> multidict.MultiDictView:
def cookies(self) -> multidict.MultiDictView[str, Tuple[str, multidict.MultiDict[str, Optional[str]]]]:
"""
The response cookies. A possibly empty
:py:class:`~mitmproxy.net.multidict.MultiDictView`, where the keys are cookie
name strings, and values are (value, attr) tuples. Value is a string,
and attr is an MultiDictView containing cookie attributes. Within
attrs, unary attributes (e.g. HTTPOnly) are indicated by a Null value.
The response cookies. A possibly empty `MultiDictView`, where the keys are cookie
name strings, and values are `(cookie value, attributes)` tuples. Within
attributes, unary attributes (e.g. `HTTPOnly`) are indicated by a `None` value.
Modifications to the MultiDictView update `Response.headers`, and vice versa.
Caveats:
Updating the attr
*Warning:* Changes to `attributes` will not be picked up unless you also reassign
the `(cookie value, attributes)` tuple directly in the `MultiDictView`.
"""
return multidict.MultiDictView(
self._get_cookies,
@ -1074,8 +1123,8 @@ class Response(Message):
This fairly complex and heuristic function refreshes a server
response for replay.
- It adjusts date, expires and last-modified headers.
- It adjusts cookie expiration.
- It adjusts date, expires, and last-modified headers.
- It adjusts cookie expiration.
"""
if not now:
now = time.time()
@ -1108,19 +1157,17 @@ class HTTPFlow(flow.Flow):
transaction.
"""
request: Request
"""The client's HTTP request."""
response: Optional[Response] = None
"""The server's HTTP response."""
error: Optional[flow.Error] = None
"""
A connection or protocol error affecting this flow.
Note that it's possible for a Flow to have both a response and an error
object. This might happen, for instance, when a response was received
from the server, but there was an error sending it back to the client.
"""
server_conn: connection.Server
client_conn: connection.Client
intercepted: bool = False
""" Is this flow currently being intercepted? """
mode: str
""" What mode was the proxy layer in when receiving this request? """
def __init__(self, client_conn, server_conn, live=None, mode="regular"):
super().__init__("http", client_conn, server_conn, live)
@ -1144,6 +1191,7 @@ class HTTPFlow(flow.Flow):
@property
def timestamp_start(self) -> float:
"""*Read-only:* An alias for `Request.timestamp_start`."""
return self.request.timestamp_start
def copy(self):

21
mitmproxy/net/server_spec.py
View File

@ -1,5 +1,5 @@
"""
Parse scheme, host and port from a string.
Server specs are used to describe an upstream proxy or server.
"""
import functools
import re
@ -31,12 +31,12 @@ def parse(server_spec: str) -> ServerSpec:
"""
Parses a server mode specification, e.g.:
- http://example.com/
- example.org
- example.com:443
- http://example.com/
- example.org
- example.com:443
Raises:
ValueError, if the server specification is invalid.
*Raises:*
- ValueError, if the server specification is invalid.
"""
m = server_spec_re.match(server_spec)
if not m:
@ -71,13 +71,10 @@ def parse(server_spec: str) -> ServerSpec:
def parse_with_mode(mode: str) -> Tuple[str, ServerSpec]:
"""
Parse a proxy mode specification, which is usually just (reverse|upstream):server-spec
Parse a proxy mode specification, which is usually just `(reverse|upstream):server-spec`.
Returns:
A (mode, server_spec) tuple.
Raises:
ValueError, if the specification is invalid.
*Raises:*
- ValueError, if the specification is invalid.
"""
mode, server_spec = mode.split(":", maxsplit=1)
return mode, parse(server_spec)

4
mitmproxy/proxy/server_hooks.py
View File

@ -26,8 +26,12 @@ class ClientDisconnectedHook(commands.StartHook):
@dataclass
class ServerConnectionHookData:
"""Event data for server connection event hooks."""
server: connection.Server
"""The server connection this hook is about."""
client: connection.Client
"""The client on the other end."""
@dataclass

14
mitmproxy/tcp.py
View File

@ -1,5 +1,4 @@
import time
from typing import List
from mitmproxy import flow
@ -7,6 +6,12 @@ from mitmproxy.coretypes import serializable
class TCPMessage(serializable.Serializable):
"""
An individual TCP "message".
Note that TCP is *stream-based* and not *message-based*.
For practical purposes the stream is chunked into messages here,
but you should not rely on message boundaries.
"""
def __init__(self, from_client, content, timestamp=None):
self.from_client = from_client
@ -31,7 +36,6 @@ class TCPMessage(serializable.Serializable):
class TCPFlow(flow.Flow):
"""
A TCPFlow is a simplified representation of a TCP session.
"""
@ -45,3 +49,9 @@ class TCPFlow(flow.Flow):
def __repr__(self):
return "<TCPFlow ({} messages)>".format(len(self.messages))
__all__ = [
"TCPFlow",
"TCPMessage",
]

67
mitmproxy/websocket.py
View File

@ -1,35 +1,54 @@
import time
"""
*Deprecation Notice:* Mitmproxy's WebSocket API is going to change soon,
see <https://github.com/mitmproxy/mitmproxy/issues/4425>.
"""
import queue
import time
import warnings
from typing import List, Optional, Union
from typing import List
from typing import Optional
from typing import Union
from mitmproxy import flow
from mitmproxy.coretypes import serializable
from mitmproxy.net import websocket
from mitmproxy.utils import human
from mitmproxy.utils import strutils
from wsproto.frame_protocol import CloseReason
from wsproto.frame_protocol import Opcode
from mitmproxy import flow
from mitmproxy.net import websocket
from mitmproxy.coretypes import serializable
from mitmproxy.utils import strutils, human
class WebSocketMessage(serializable.Serializable):
"""
A WebSocket message sent from one endpoint to the other.
"""
type: Opcode
"""indicates either TEXT or BINARY (from wsproto.frame_protocol.Opcode)."""
from_client: bool
"""True if this messages was sent by the client."""
content: Union[bytes, str]
"""A byte-string representing the content of this message."""
timestamp: float
"""Timestamp of when this message was received or created."""
killed: bool
"""True if this messages was killed and should not be sent to the other endpoint."""
def __init__(
self, type: int, from_client: bool, content: Union[bytes, str], timestamp: Optional[float]=None, killed: bool=False
self,
type: int,
from_client: bool,
content: Union[bytes, str],
timestamp: Optional[float] = None,
killed: bool = False
) -> None:
self.type = Opcode(type) # type: ignore
"""indicates either TEXT or BINARY (from wsproto.frame_protocol.Opcode)."""
self.from_client = from_client
"""True if this messages was sent by the client."""
self.content = content
"""A byte-string representing the content of this message."""
self.timestamp: float = timestamp or time.time()
"""Timestamp of when this message was received or created."""
self.killed = killed
"""True if this messages was killed and should not be sent to the other endpoint."""
@classmethod
def from_state(cls, state):
@ -147,25 +166,3 @@ class WebSocketFlow(flow.Flow):
direction="->" if message.from_client else "<-",
endpoint=self.handshake_flow.request.path,
)
def inject_message(self, endpoint, payload):
"""
Inject and send a full WebSocket message to the remote endpoint.
This might corrupt your WebSocket connection! Be careful!
The endpoint needs to be either flow.client_conn or flow.server_conn.
If ``payload`` is of type ``bytes`` then the message is flagged as
being binary If it is of type ``str`` encoded as UTF-8 and sent as
text.
:param payload: The message body to send.
:type payload: ``bytes`` or ``str``
"""
if endpoint == self.client_conn:
self._inject_messages_client.put(payload)
elif endpoint == self.server_conn:
self._inject_messages_server.put(payload)
else:
raise ValueError('Invalid endpoint')

4
setup.cfg
View File

@ -36,6 +36,10 @@ ignore_errors = True
[mypy-test.*]
ignore_errors = True
# https://github.com/python/mypy/issues/3004
[mypy-http-modify-form,http-trailers]
ignore_errors = True
[tool:full_coverage]
exclude =
mitmproxy/tools/

1
setup.py
View File

@ -99,6 +99,7 @@ setup(
'dev': [
"hypothesis>=5.8,<6.2",
"parver>=0.1,<2.0",
"pdoc>=4.0.0",
"pytest-asyncio>=0.10.0,<0.14,!=0.14",
"pytest-cov>=2.7.1,<3",
"pytest-timeout>=1.3.3,<2",

2
test/examples/test_examples.py
View File

@ -10,7 +10,7 @@ from ..mitmproxy import tservers
class TestScripts(tservers.MasterTest):
def test_add_header(self, tdata):
with taddons.context() as tctx:
a = tctx.script(tdata.path("../examples/addons/scripting-minimal-example.py"))
a = tctx.script(tdata.path("../examples/addons/anatomy2.py"))
f = tflow.tflow()
a.request(f)
assert f.request.headers["myheader"] == "value"

22
test/mitmproxy/test_http.py
View File

@ -751,7 +751,7 @@ class TestHeaders:
headers = Headers()
assert len(headers) == 0
headers = Headers([[b"Host", b"example.com"]])
headers = Headers([(b"Host", b"example.com")])
assert len(headers) == 1
assert headers["Host"] == "example.com"
@ -760,14 +760,14 @@ class TestHeaders:
assert headers["Host"] == "example.com"
headers = Headers(
[[b"Host", b"invalid"]],
[(b"Host", b"invalid")],
Host="example.com"
)
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(
[[b"Host", b"invalid"], [b"Accept", b"text/plain"]],
[(b"Host", b"invalid"), (b"Accept", b"text/plain")],
Host="example.com"
)
assert len(headers) == 2
@ -775,7 +775,7 @@ class TestHeaders:
assert headers["Accept"] == "text/plain"
with pytest.raises(TypeError):
Headers([[b"Host", "not-bytes"]])
Headers([(b"Host", "not-bytes")])
def test_set(self):
headers = Headers()
@ -791,8 +791,8 @@ class TestHeaders:
assert bytes(headers) == b"Host: example.com\r\n"
headers = Headers([
[b"Host", b"example.com"],
[b"Accept", b"text/plain"]
(b"Host", b"example.com"),
(b"Accept", b"text/plain")
])
assert bytes(headers) == b"Host: example.com\r\nAccept: text/plain\r\n"
@ -801,8 +801,8 @@ class TestHeaders:
def test_iter(self):
headers = Headers([
[b"Set-Cookie", b"foo"],
[b"Set-Cookie", b"bar"]
(b"Set-Cookie", b"foo"),
(b"Set-Cookie", b"bar")
])
assert list(headers) == ["Set-Cookie"]
@ -816,9 +816,9 @@ class TestHeaders:
def test_items(self):
headers = Headers([
[b"Set-Cookie", b"foo"],
[b"Set-Cookie", b"bar"],
[b'Accept', b'text/plain'],
(b"Set-Cookie", b"foo"),
(b"Set-Cookie", b"bar"),
(b'Accept', b'text/plain'),
])
assert list(headers.items()) == [
('Set-Cookie', 'foo, bar'),

12
test/mitmproxy/test_websocket.py
View File

@ -86,15 +86,3 @@ class TestWebSocketFlow:
d = tflow.twebsocketflow().handshake_flow.get_state()
tnetstring.dump(d, b)
assert b.getvalue()
def test_inject_message(self):
f = tflow.twebsocketflow()
with pytest.raises(ValueError):
f.inject_message(None, 'foobar')
f.inject_message(f.client_conn, 'foobar')
assert f._inject_messages_client.qsize() == 1
f.inject_message(f.server_conn, 'foobar')
assert f._inject_messages_client.qsize() == 1