Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-22 07:08:10 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4422 from mhils/pdoc

Browse Source 
Docs: Add API Reference Using Pdoc
This commit is contained in:
Maximilian Hils 2021-02-13 00:18:49 +01:00 committed by GitHub
commit 748fc93699
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
50 changed files with 1043 additions and 695 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs/build.py Executable file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env python3
import shutil
import subprocess
from pathlib import Path
here = Path(__file__).parent
for script in (here / "scripts").glob("*.py"):
print(f"Generating output for {script.name}...")
out = subprocess.check_output(["python3", script.absolute()], text=True)
if out:
(here / "src" / "generated" / f"{script.stem}.html").write_text(out, encoding="utf8")
if (here / "public").exists():
shutil.rmtree(here / "public")
subprocess.run(["hugo"], cwd=here / "src")

20
docs/build.sh
View File

@ -1,20 +0,0 @@
#!/usr/bin/env bash
set -o errexit
set -o pipefail
set -o nounset
# set -o xtrace
SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
pushd ${SCRIPTPATH}
for script in scripts/*.py ; do
output="${script##*/}"
output="src/generated/${output%.*}.html"
echo "Generating output for ${script} into ${output} ..."
"${script}" > "${output}"
done
rm -rf ./public
cd src
hugo

2
docs/ci.sh
View File

@ -6,7 +6,7 @@ set -o pipefail
# This script gets run from CI to render and upload docs for the master branch. # This script gets run from CI to render and upload docs for the master branch.
./build.sh ./build.py
# Only upload if we have defined credentials - we only have these defined for # Only upload if we have defined credentials - we only have these defined for
# trusted commits (i.e. not PRs). # trusted commits (i.e. not PRs).

4
docs/modd.conf
View File

@ -1,5 +1,5 @@
scripts/*.py { scripts/** {
prep: ./build.sh prep: python3 build.py
} }
{ {

151
docs/scripts/api-events.py Normal file
View File

@ -0,0 +1,151 @@
#!/usr/bin/env python3
import contextlib
import inspect
import textwrap
from pathlib import Path
from typing import List, Type
import mitmproxy.addons.next_layer # noqa
from mitmproxy import hooks, log, addonmanager
from mitmproxy.proxy import server_hooks, layer
from mitmproxy.proxy.layers import http, tcp, tls, websocket
known = set()
def category(name: str, desc: str, hooks: List[Type[hooks.Hook]]) -> None:
all_params = [
list(inspect.signature(hook.__init__).parameters.values())[1:]
for hook in hooks
]
# slightly overengineered, but this was fun to write. ¯\_(ツ)_/¯
imports = set()
types = set()
for params in all_params:
for param in params:
try:
mod = inspect.getmodule(param.annotation).__name__
if mod == "typing":
# this is ugly, but can be removed once we are on Python 3.9+ only
imports.add(inspect.getmodule(param.annotation.__args__[0]).__name__)
types.add(param.annotation._name)
else:
imports.add(mod)
except AttributeError:
raise ValueError(f"Missing type annotation: {params}")
imports.discard("builtins")
if types:
print(f"from typing import {', '.join(sorted(types))}")
print("from mitmproxy import ctx")
for imp in sorted(imports):
print(f"import {imp}")
print()
print(f"class {name}Events:")
print(f' """{desc}"""')
first = True
for hook, params in zip(hooks, all_params):
if first:
first = False
else:
print()
if hook.name in known:
raise RuntimeError(f"Already documented: {hook}")
known.add(hook.name)
doc = inspect.getdoc(hook)
print(f" def {hook.name}({', '.join(str(p) for p in ['self'] + params)}):")
print(textwrap.indent(f'"""\n{doc}\n"""', " "))
if params:
print(f' ctx.log(f"{hook.name}: {" ".join("{" + p.name + "=}" for p in params)}")')
else:
print(f' ctx.log("{hook.name}")')
print("")
outfile = Path(__file__).parent.parent / "src" / "generated" / "events.py"
with outfile.open("w") as f, contextlib.redirect_stdout(f):
print("# This file is autogenerated, do not edit manually.")
category(
"Lifecycle",
"",
[
addonmanager.LoadHook,
hooks.RunningHook,
hooks.ConfigureHook,
hooks.DoneHook,
]
)
category(
"Connection",
"",
[
server_hooks.ClientConnectedHook,
server_hooks.ClientDisconnectedHook,
server_hooks.ServerConnectHook,
server_hooks.ServerConnectedHook,
server_hooks.ServerDisconnectedHook,
]
)
category(
"HTTP",
"",
[
http.HttpRequestHeadersHook,
http.HttpRequestHook,
http.HttpResponseHeadersHook,
http.HttpResponseHook,
http.HttpErrorHook,
http.HttpConnectHook,
]
)
category(
"TCP",
"",
[
tcp.TcpStartHook,
tcp.TcpMessageHook,
tcp.TcpEndHook,
tcp.TcpErrorHook,
]
)
category(
"TLS",
"",
[
tls.TlsClienthelloHook,
tls.TlsStartHook,
]
)
category(
"WebSocket",
"",
[
websocket.WebsocketStartHook,
websocket.WebsocketMessageHook,
websocket.WebsocketEndHook,
websocket.WebsocketErrorHook,
]
)
category(
"AdvancedLifecycle",
"",
[
layer.NextLayerHook,
hooks.UpdateHook,
log.AddLogHook,
]
)
not_documented = set(hooks.all_hooks.keys()) - known
if not_documented:
raise RuntimeError(f"Not documented: {not_documented}")

67
docs/scripts/api-render.py Normal file
View File

@ -0,0 +1,67 @@
#!/usr/bin/env python3
import os
import shutil
import textwrap
from pathlib import Path
import pdoc.render_helpers
here = Path(__file__).parent
if os.environ.get("DOCS_ARCHIVE", False):
edit_url_map = {}
else:
edit_url_map = {
"mitmproxy": "https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/",
}
pdoc.render.configure(
template_directory=here / "pdoc-template",
edit_url_map=edit_url_map,
)
# We can't configure Hugo, but we can configure pdoc.
pdoc.render_helpers.formatter.cssclass = "chroma"
modules = [
"mitmproxy.addonmanager",
"mitmproxy.certs",
"mitmproxy.connection",
"mitmproxy.coretypes.multidict",
"mitmproxy.flow",
"mitmproxy.http",
"mitmproxy.net.server_spec",
"mitmproxy.proxy.server_hooks",
"mitmproxy.tcp",
"mitmproxy.websocket",
here / ".." / "src" / "generated" / "events.py",
]
pdoc.pdoc(
*modules,
output_directory=here / ".." / "src" / "generated" / "api"
)
api_content = here / ".." / "src" / "content" / "api"
if api_content.exists():
shutil.rmtree(api_content)
api_content.mkdir()
for module in modules:
if isinstance(module, Path):
continue
filename = f"api/{module.replace('.', '/')}.html"
(api_content / f"{module}.md").write_text(textwrap.dedent(f"""
---
title: "{module}"
url: "{filename}"
menu:
addons:
parent: 'Event Hooks & API'
---
{{{{< readfile file="/generated/{filename}" >}}}}
"""))
(here / ".." / "src" / "content" / "addons-api.md").touch()

142
docs/scripts/events.py
View File

@ -1,142 +0,0 @@
#!/usr/bin/env python3
import inspect
import textwrap
from typing import List, Type
import mitmproxy.addons.next_layer # noqa
from mitmproxy import hooks, log, addonmanager
from mitmproxy.proxy import server_hooks, layer
from mitmproxy.proxy.layers import http, tcp, tls, websocket
known = set()
def category(name: str, hooks: List[Type[hooks.Hook]]) -> None:
print(f"### {name} Events")
print("```python")
all_params = [
list(inspect.signature(hook.__init__).parameters.values())[1:]
for hook in hooks
]
# slightly overengineered, but this was fun to write. ¯\_(ツ)_/¯
imports = set()
types = set()
for params in all_params:
for param in params:
try:
mod = inspect.getmodule(param.annotation).__name__
if mod == "typing":
# this is ugly, but can be removed once we are on Python 3.9+ only
imports.add(inspect.getmodule(param.annotation.__args__[0]).__name__)
types.add(param.annotation._name)
else:
imports.add(mod)
except AttributeError:
raise ValueError(f"Missing type annotation: {params}")
imports.discard("builtins")
if types:
print(f"from typing import {', '.join(sorted(types))}")
print("from mitmproxy import ctx")
for imp in sorted(imports):
print(f"import {imp}")
print()
first = True
for hook, params in zip(hooks, all_params):
if first:
first = False
else:
print()
if hook.name in known:
raise RuntimeError(f"Already documented: {hook}")
known.add(hook.name)
doc = inspect.getdoc(hook)
print(f"def {hook.name}({', '.join(str(p) for p in params)}):")
print(textwrap.indent(f'"""\n{doc}\n"""', " "))
if params:
print(f' ctx.log(f"{hook.name}: {" ".join("{" + p.name + "=}" for p in params)}")')
else:
print(f' ctx.log("{hook.name}")')
print("```")
category(
"Lifecycle",
[
addonmanager.LoadHook,
hooks.RunningHook,
hooks.ConfigureHook,
hooks.DoneHook,
]
)
category(
"Connection",
[
server_hooks.ClientConnectedHook,
server_hooks.ClientDisconnectedHook,
server_hooks.ServerConnectHook,
server_hooks.ServerConnectedHook,
server_hooks.ServerDisconnectedHook,
]
)
category(
"HTTP",
[
http.HttpRequestHeadersHook,
http.HttpRequestHook,
http.HttpResponseHeadersHook,
http.HttpResponseHook,
http.HttpErrorHook,
http.HttpConnectHook,
]
)
category(
"TCP",
[
tcp.TcpStartHook,
tcp.TcpMessageHook,
tcp.TcpEndHook,
tcp.TcpErrorHook,
]
)
category(
"TLS",
[
tls.TlsClienthelloHook,
tls.TlsStartHook,
]
)
category(
"WebSocket",
[
websocket.WebsocketStartHook,
websocket.WebsocketMessageHook,
websocket.WebsocketEndHook,
websocket.WebsocketErrorHook,
]
)
category(
"Advanced Lifecycle",
[
layer.NextLayerHook,
hooks.UpdateHook,
log.AddLogHook,
]
)
not_documented = set(hooks.all_hooks.keys()) - known
if not_documented:
raise RuntimeError(f"Not documented: {not_documented}")
# print("<table class=\"table filtertable\"><tbody>")
# for i in flowfilter.help:
# print("<tr><th>%s</th><td>%s</td></tr>" % i)
# print("</tbody></table>")

31
docs/scripts/examples.py
View File

@ -28,21 +28,38 @@ for example in examples:
else: else:
comment = "" comment = ""
overview.append( overview.append(
f" * [{example.name}](#{slug}){comment}" f" * [{example.name}](#{slug}){comment}\n"
) )
listings.append(f""" listings.append(f"""
<h2 id="{slug}">Example: {example.name}</h2> <h3 id="{slug}">Example: {example.name}</h3>
```python ```python
{code} {code.strip()}
``` ```
""") """)
print("\n".join(overview))
print(""" print(f"""
### Community Examples # Addon Examples
### Dedicated Example Addons
{"".join(overview)}
### Built-In Addons
Much of mitmproxy’s own functionality is defined in
[a suite of built-in addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons),
implementing everything from functionality like anticaching and sticky cookies to our onboarding webapp.
The built-in addons make for instructive reading, and you will quickly see that quite complex functionality
can often boil down to a very small, completely self-contained modules.
### Additional Community Examples
Additional examples contributed by the mitmproxy community can be found Additional examples contributed by the mitmproxy community can be found
[on GitHub](https://github.com/mitmproxy/mitmproxy/tree/master/examples/contrib). [on GitHub](https://github.com/mitmproxy/mitmproxy/tree/master/examples/contrib).
-------------------------
{"".join(listings)}
""") """)
print("\n".join(listings))

3
docs/scripts/pdoc-template/frame.html.jinja2 Normal file
View File

@ -0,0 +1,3 @@
{% block style %}{% endblock %}
{% block body %}{% endblock %}
<div class="pdoc" style="margin-top: 4rem">{% block attribution %}{% endblock %}</div>

62
docs/scripts/pdoc-template/module.html.jinja2 Normal file
View File

@ -0,0 +1,62 @@
{% extends "default/module.html.jinja2" %}
{% block nav %}{% endblock %}
{% block style_layout %}{% endblock %}
{% block style_pygments %}{% endblock %}
{#
To document all event hooks, we do a bit of hackery:
1. scripts/api-events.py auto-generates generated/events.py.
2. scripts/api-render.py renders generated/events.py together with the remaining API docs.
3. This templates hides some elements of the default pdoc template.
#}
{% if module.name == "events" %}
{% macro module_name() %}
{% endmacro %}
{% macro view_source(doc) %}
{% if doc.type != "module" %}
{{ default_view_source(doc) }}
{% endif %}
{% endmacro %}
{% macro is_public(doc) %}
{% if doc.name != "__init__" %}
{{ default_is_public(doc) }}
{% endif %}
{% endmacro %}
{% else %}
{% macro is_public(doc) %}
{% if doc.name is in(["from_state", "get_state", "set_state"]) %}
{% elif doc.modulename == "mitmproxy.addonmanager" %}
{% if doc.qualname.startswith("Loader") and not doc.name.startswith("_") %}
true
{% endif %}
{% elif doc.modulename == "mitmproxy.certs" %}
{% if doc.qualname == "Cert" or doc.qualname.startswith("Cert.") %}
{{ default_is_public(doc) }}
{% endif %}
{% elif doc.modulename == "mitmproxy.coretypes.multidict" %}
{% if doc.name == "_MultiDict" %}
true
{% else %}
{{ default_is_public(doc) }}
{% endif %}
{% elif doc.modulename == "mitmproxy.flow" %}
{% if doc.name is not in(["__init__", "reply", "metadata"]) %}
{{ default_is_public(doc) }}
{% endif %}
{% elif doc.modulename == "mitmproxy.http" %}
{% if doc.qualname is not in([
"Message.__init__", "Message.data",
"Request.data",
"Response.data",
]) %}
{{ default_is_public(doc) }}
{% endif %}
{% elif doc.modulename == "mitmproxy.proxy.server_hooks" %}
{% if doc.qualname.startswith("ServerConnectionHookData") and doc.name != "__init__" %}
{{ default_is_public(doc) }}
{% endif %}
{% else %}
{{ default_is_public(doc) }}
{% endif %}
{% endmacro %}
{% endif %}

15
docs/src/assets/style.scss
View File

@ -1,4 +1,12 @@
@import "./syntax"; @import "./syntax";
/* background for both hugo *and* pdoc. */
.chroma pre, pre.chroma {
background-color: #f7f7f7;
border-top: 1px solid #ccc;
border-bottom: 1px solid #ccc;
padding: .5rem 0 .5rem .5rem;
}
@import "./badge"; @import "./badge";
$primary: #C93312; $primary: #C93312;
@ -7,6 +15,9 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox
$panel-heading-size: 1em; $panel-heading-size: 1em;
$panel-heading-weight: 600; $panel-heading-weight: 600;
$menu-list-link-padding: .3em .75em;
$menu-label-spacing: .7em;
$menu-nested-list-margin: .3em .75em;
/*!* /*!*
bulma.io v0.8.0 | MIT License | github.com/jgthms/bulma */ bulma.io v0.8.0 | MIT License | github.com/jgthms/bulma */
@ -17,6 +28,10 @@ bulma.io v0.8.0 | MIT License | github.com/jgthms/bulma */
@import "./bulma/components/_all"; @import "./bulma/components/_all";
@import "./bulma/layout/_all"; @import "./bulma/layout/_all";
html {
scroll-behavior: smooth;
}
html, body { html, body {
height: 100%; height: 100%;
} }

135
docs/src/assets/syntax.css
View File

@ -1,59 +1,82 @@
/* Background */ .chroma { color: #f8f8f2; background-color: #272822 } /* Background */ .chroma { background-color: #ffffff }
/* Error */ .chroma .err { color: #960050; background-color: #1e0010 } /* Other */ .chroma .x { }
/* Error */ .chroma .err { color: #a61717; background-color: #e3d2d2 }
/* LineTableTD */ .chroma .lntd { vertical-align: top; padding: 0; margin: 0; border: 0; } /* LineTableTD */ .chroma .lntd { vertical-align: top; padding: 0; margin: 0; border: 0; }
/* LineTable */ .chroma .lntable { border-spacing: 0; padding: 0; margin: 0; border: 0; width: 100%; overflow: auto; display: block; } /* LineTable */ .chroma .lntable { border-spacing: 0; padding: 0; margin: 0; border: 0; width: auto; overflow: auto; display: block; }
/* LineHighlight */ .chroma .hl { display: block; width: 100%;background-color: #ffffcc } /* LineHighlight */ .chroma .hl { display: block; width: 100%;background-color: #ffffcc }
/* LineNumbersTable */ .chroma .lnt { margin-right: 0.4em; padding: 0 0.4em 0 0.4em; display: block; } /* LineNumbersTable */ .chroma .lnt { margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #7f7f7f }
/* LineNumbers */ .chroma .ln { margin-right: 0.4em; padding: 0 0.4em 0 0.4em; } /* LineNumbers */ .chroma .ln { margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #7f7f7f }
/* Keyword */ .chroma .k { color: #66d9ef } /* Keyword */ .chroma .k { color: #000000; font-weight: bold }
/* KeywordConstant */ .chroma .kc { color: #66d9ef } /* KeywordConstant */ .chroma .kc { color: #000000; font-weight: bold }
/* KeywordDeclaration */ .chroma .kd { color: #66d9ef } /* KeywordDeclaration */ .chroma .kd { color: #000000; font-weight: bold }
/* KeywordNamespace */ .chroma .kn { color: #f92672 } /* KeywordNamespace */ .chroma .kn { color: #000000; font-weight: bold }
/* KeywordPseudo */ .chroma .kp { color: #66d9ef } /* KeywordPseudo */ .chroma .kp { color: #000000; font-weight: bold }
/* KeywordReserved */ .chroma .kr { color: #66d9ef } /* KeywordReserved */ .chroma .kr { color: #000000; font-weight: bold }
/* KeywordType */ .chroma .kt { color: #66d9ef } /* KeywordType */ .chroma .kt { color: #445588; font-weight: bold }
/* NameAttribute */ .chroma .na { color: #a6e22e } /* Name */ .chroma .n { }
/* NameClass */ .chroma .nc { color: #a6e22e } /* NameAttribute */ .chroma .na { color: #008080 }
/* NameConstant */ .chroma .no { color: #66d9ef } /* NameBuiltin */ .chroma .nb { color: #0086b3 }
/* NameDecorator */ .chroma .nd { color: #a6e22e } /* NameBuiltinPseudo */ .chroma .bp { color: #999999 }
/* NameException */ .chroma .ne { color: #a6e22e } /* NameClass */ .chroma .nc { color: #445588; font-weight: bold }
/* NameFunction */ .chroma .nf { color: #a6e22e } /* NameConstant */ .chroma .no { color: #008080 }
/* NameOther */ .chroma .nx { color: #a6e22e } /* NameDecorator */ .chroma .nd { color: #3c5d5d; font-weight: bold }
/* NameTag */ .chroma .nt { color: #f92672 } /* NameEntity */ .chroma .ni { color: #800080 }
/* Literal */ .chroma .l { color: #ae81ff } /* NameException */ .chroma .ne { color: #990000; font-weight: bold }
/* LiteralDate */ .chroma .ld { color: #e6db74 } /* NameFunction */ .chroma .nf { color: #990000; font-weight: bold }
/* LiteralString */ .chroma .s { color: #e6db74 } /* NameFunctionMagic */ .chroma .fm { }
/* LiteralStringAffix */ .chroma .sa { color: #e6db74 } /* NameLabel */ .chroma .nl { color: #990000; font-weight: bold }
/* LiteralStringBacktick */ .chroma .sb { color: #e6db74 } /* NameNamespace */ .chroma .nn { color: #555555 }
/* LiteralStringChar */ .chroma .sc { color: #e6db74 } /* NameOther */ .chroma .nx { }
/* LiteralStringDelimiter */ .chroma .dl { color: #e6db74 } /* NameProperty */ .chroma .py { }
/* LiteralStringDoc */ .chroma .sd { color: #e6db74 } /* NameTag */ .chroma .nt { color: #000080 }
/* LiteralStringDouble */ .chroma .s2 { color: #e6db74 } /* NameVariable */ .chroma .nv { color: #008080 }
/* LiteralStringEscape */ .chroma .se { color: #ae81ff } /* NameVariableClass */ .chroma .vc { color: #008080 }
/* LiteralStringHeredoc */ .chroma .sh { color: #e6db74 } /* NameVariableGlobal */ .chroma .vg { color: #008080 }
/* LiteralStringInterpol */ .chroma .si { color: #e6db74 } /* NameVariableInstance */ .chroma .vi { color: #008080 }
/* LiteralStringOther */ .chroma .sx { color: #e6db74 } /* NameVariableMagic */ .chroma .vm { }
/* LiteralStringRegex */ .chroma .sr { color: #e6db74 } /* Literal */ .chroma .l { }
/* LiteralStringSingle */ .chroma .s1 { color: #e6db74 } /* LiteralDate */ .chroma .ld { }
/* LiteralStringSymbol */ .chroma .ss { color: #e6db74 } /* LiteralString */ .chroma .s { color: #dd1144 }
/* LiteralNumber */ .chroma .m { color: #ae81ff } /* LiteralStringAffix */ .chroma .sa { color: #dd1144 }
/* LiteralNumberBin */ .chroma .mb { color: #ae81ff } /* LiteralStringBacktick */ .chroma .sb { color: #dd1144 }
/* LiteralNumberFloat */ .chroma .mf { color: #ae81ff } /* LiteralStringChar */ .chroma .sc { color: #dd1144 }
/* LiteralNumberHex */ .chroma .mh { color: #ae81ff } /* LiteralStringDelimiter */ .chroma .dl { color: #dd1144 }
/* LiteralNumberInteger */ .chroma .mi { color: #ae81ff } /* LiteralStringDoc */ .chroma .sd { color: #dd1144 }
/* LiteralNumberIntegerLong */ .chroma .il { color: #ae81ff } /* LiteralStringDouble */ .chroma .s2 { color: #dd1144 }
/* LiteralNumberOct */ .chroma .mo { color: #ae81ff } /* LiteralStringEscape */ .chroma .se { color: #dd1144 }
/* Operator */ .chroma .o { color: #f92672 } /* LiteralStringHeredoc */ .chroma .sh { color: #dd1144 }
/* OperatorWord */ .chroma .ow { color: #f92672 } /* LiteralStringInterpol */ .chroma .si { color: #dd1144 }
/* Comment */ .chroma .c { color: #75715e } /* LiteralStringOther */ .chroma .sx { color: #dd1144 }
/* CommentHashbang */ .chroma .ch { color: #75715e } /* LiteralStringRegex */ .chroma .sr { color: #009926 }
/* CommentMultiline */ .chroma .cm { color: #75715e } /* LiteralStringSingle */ .chroma .s1 { color: #dd1144 }
/* CommentSingle */ .chroma .c1 { color: #75715e } /* LiteralStringSymbol */ .chroma .ss { color: #990073 }
/* CommentSpecial */ .chroma .cs { color: #75715e } /* LiteralNumber */ .chroma .m { color: #009999 }
/* CommentPreproc */ .chroma .cp { color: #75715e } /* LiteralNumberBin */ .chroma .mb { color: #009999 }
/* CommentPreprocFile */ .chroma .cpf { color: #75715e } /* LiteralNumberFloat */ .chroma .mf { color: #009999 }
/* GenericDeleted */ .chroma .gd { color: #f92672 } /* LiteralNumberHex */ .chroma .mh { color: #009999 }
/* GenericEmph */ .chroma .ge { font-style: italic } /* LiteralNumberInteger */ .chroma .mi { color: #009999 }
/* GenericInserted */ .chroma .gi { color: #a6e22e } /* LiteralNumberIntegerLong */ .chroma .il { color: #009999 }
/* LiteralNumberOct */ .chroma .mo { color: #009999 }
/* Operator */ .chroma .o { color: #000000; font-weight: bold }
/* OperatorWord */ .chroma .ow { color: #000000; font-weight: bold }
/* Punctuation */ .chroma .p { }
/* Comment */ .chroma .c { color: #999988; font-style: italic }
/* CommentHashbang */ .chroma .ch { color: #999988; font-style: italic }
/* CommentMultiline */ .chroma .cm { color: #999988; font-style: italic }
/* CommentSingle */ .chroma .c1 { color: #999988; font-style: italic }
/* CommentSpecial */ .chroma .cs { color: #999999; font-weight: bold; font-style: italic }
/* CommentPreproc */ .chroma .cp { color: #999999; font-weight: bold; font-style: italic }
/* CommentPreprocFile */ .chroma .cpf { color: #999999; font-weight: bold; font-style: italic }
/* Generic */ .chroma .g { }
/* GenericDeleted */ .chroma .gd { color: #000000; background-color: #ffdddd }
/* GenericEmph */ .chroma .ge { color: #000000; font-style: italic }
/* GenericError */ .chroma .gr { color: #aa0000 }
/* GenericHeading */ .chroma .gh { color: #999999 }
/* GenericInserted */ .chroma .gi { color: #000000; background-color: #ddffdd }
/* GenericOutput */ .chroma .go { color: #888888 }
/* GenericPrompt */ .chroma .gp { color: #555555 }
/* GenericStrong */ .chroma .gs { font-weight: bold } /* GenericStrong */ .chroma .gs { font-weight: bold }
/* GenericSubheading */ .chroma .gu { color: #75715e } /* GenericSubheading */ .chroma .gu { color: #aaaaaa }
/* GenericTraceback */ .chroma .gt { color: #aa0000 }
/* GenericUnderline */ .chroma .gl { text-decoration: underline }
/* TextWhitespace */ .chroma .w { color: #bbbbbb }

1
docs/src/config.toml
View File

@ -5,6 +5,7 @@ theme = "mitmproxydocs"
publishDir = "../public" publishDir = "../public"
RelativeURLs = true RelativeURLs = true
pygmentsCodefences = true pygmentsCodefences = true
pygmentsUseClasses = true
[indexes] [indexes]
tag = "tags" tag = "tags"

15
docs/src/content/addons-events.md → docs/src/content/addons-api.md
View File

@ -1,8 +1,12 @@
--- ---
title: "Event Hooks" title: "Event Hooks & API"
url: "api/events.html"
aliases:
- /addons-events/
layout: single
menu: menu:
addons: addons:
weight: 2 weight: 3
--- ---
# Event Hooks # Event Hooks
@ -16,9 +20,8 @@ header with a count of the number of responses seen:
{{< example src="examples/addons/http-add-header.py" lang="py" >}} {{< example src="examples/addons/http-add-header.py" lang="py" >}}
## Supported Events ## Available Hooks
Below we list events supported by mitmproxy. We've added The following addons list all available event hooks.
annotations to illustrate the argument types.
{{< readfile file="/generated/events.html" markdown="true" >}} {{< readfile file="/generated/api/events.html" >}}

4
docs/src/content/addons-examples.md
View File

@ -1,11 +1,9 @@
--- ---
title: "Example Addons" title: "Examples"
menu: menu:
addons: addons:
weight: 6 weight: 6
--- ---
# Example Addons
{{< readfile file="/generated/examples.html" markdown="true" >}} {{< readfile file="/generated/examples.html" markdown="true" >}}

61
docs/src/content/addons-overview.md
View File

@ -7,39 +7,19 @@ menu:
# Addons # Addons
Mitmproxy's addon mechanism consists of a set of APIs that support components of Mitmproxy's addon mechanism is an exceptionally powerful part of mitmproxy. In fact, much of mitmproxy's own
any complexity. Addons interact with mitmproxy by responding to **events**, functionality is defined in
which allow them to hook into and change mitmproxy's behaviour. They are [a suite of built-in addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons),
configured through **[options]({{< relref concepts-options >}})**, which can be implementing everything from functionality like
set in mitmproxy's config file, changed interactively by users, or passed on the [anticaching]({{< relref "overview-features#anticache" >}}) and [sticky cookies]({{< relref
command-line. Finally, they can expose **commands**, which allows users to "overview-features#sticky-cookies" >}}) to our onboarding webapp.
invoke their actions either directly or by binding them to keys in the
interactive tools.
Addons are an exceptionally powerful part of mitmproxy. In fact, much of Addons interact with mitmproxy by responding to [events]({{< relref addons-api >}}), which allow them to hook into and
mitmproxy's own functionality is defined in [a suite of built-in change mitmproxy's behaviour. They are configured through [options]({{< relref addons-options >}}), which can be set in
addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons), mitmproxy's config file, changed interactively by users, or passed on the command-line. Finally, they can expose
implementing everything from functionality like [anticaching]({{< relref [commands]({{< relref addons-commands >}}), which allows users to invoke their actions either directly or by binding
"overview-features#anticache" >}}) and [sticky cookies]({{< relref them to keys in the interactive tools.
"overview-features#sticky-cookies" >}}) to our onboarding webapp. The built-in
addons make for instructive reading, and you will quickly see that quite complex
functionality can often boil down to a very small, completely self-contained
modules. Mitmproxy provides the exact same set of facilities it uses for its own
functionality to third-party scripters and extenders.
This document will show you how to build addons using **events**, **options**
and **commands**. However, this is not an API manual, and the mitmproxy source
code remains the canonical reference. One easy way to explore the API from the
command-line is to use [pydoc](https://docs.python.org/3/library/pydoc.html).
Here, for example, is a command that shows the API documentation for the
mitmproxy's HTTP flow classes:
```bash
pydoc mitmproxy.http
```
You will be referring to the mitmproxy API documentation frequently, so keep
**pydoc** or an equivalent handy.
# Anatomy of an addon # Anatomy of an addon
@ -55,7 +35,7 @@ it into your mitmproxy tool of choice. We'll use mitmpdump in these examples,
but the flag is identical for all tools: but the flag is identical for all tools:
```bash ```bash
> mitmdump -s ./anatomy.py mitmdump -s ./anatomy.py
``` ```
Here are a few things to note about the code above: Here are a few things to note about the code above:
@ -63,12 +43,21 @@ Here are a few things to note about the code above:
- Mitmproxy picks up the contents of the `addons` global list and loads what it - Mitmproxy picks up the contents of the `addons` global list and loads what it
finds into the addons mechanism. finds into the addons mechanism.
- Addons are just objects - in this case our addon is an instance of `Counter`. - Addons are just objects - in this case our addon is an instance of `Counter`.
- The `request` method is an example of an **event**. Addons simply implement a - The `request` method is an example of an *event*. Addons simply implement a
method for each event they want to handle. Each event has a signature method for each event they want to handle. Each event and its signature are documented
consisting of arguments that are passed to the method. For `request`, this is in the [API documentation]({{< relref "addons-api" >}}).
an instance of `mitmproxy.http.HTTPFlow`.
- Finally, the `ctx` module is a holdall module that exposes a set of standard - Finally, the `ctx` module is a holdall module that exposes a set of standard
objects that are commonly used in addons. We could pass a `ctx` object as the objects that are commonly used in addons. We could pass a `ctx` object as the
first parameter to every event, but we've found it neater to just expose it as first parameter to every event, but we've found it neater to just expose it as
an importable global. In this case, we're using the `ctx.log` object to do our an importable global. In this case, we're using the `ctx.log` object to do our
logging. logging.
# Abbreviated Scripting Syntax
Sometimes, we would like to write a quick script without going through the trouble of creating a class.
The addons mechanism has a shorthand that allows a module as a whole to be treated as an addon object.
This lets us place event handler functions in the module scope.
For instance, here is a complete script that adds a header to every request:
{{< example src="examples/addons/anatomy2.py" lang="py" >}}

51
docs/src/content/addons-scripting.md
View File

@ -1,51 +0,0 @@
---
title: "Scripting"
menu:
addons:
weight: 5
---
# Scripting HTTP/1.1 and HTTP/2.0
Sometimes, we would like to write a quick script without going through the
trouble of creating a class. The addons mechanism has a shorthand that allows a
module as a whole to be treated as an addon object. This lets us place event
handler functions in the module scope. For instance, here is a complete script
that adds a header to every request.
{{< example src="examples/addons/scripting-minimal-example.py" lang="py" >}}
Here's another example that intercepts requests to a particular URL and sends
an arbitrary response instead:
{{< example src="examples/addons/http-reply-from-proxy.py" lang="py" >}}
All events around the HTTP protocol [can be found here]({{< relref "addons-events#http-events">}}).
For HTTP-related objects, please look at the [http][] module, or the
[Request][], and [Response][] classes for other attributes that you can use when
scripting.
# Scripting WebSocket
The WebSocket protocol initially looks like a regular HTTP request, before the client and server agree to upgrade the connection to WebSocket. All scripting events for initial HTTP handshake, and also the dedicated WebSocket events [can be found here]({{< relref "addons-events#websocket-events">}}).
{{< example src="examples/addons/websocket-simple.py" lang="py" >}}
For WebSocket-related objects please look at the [websocket][] module to find
all attributes that you can use when scripting.
[websocket]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/websocket.py
# Scripting TCP
All events around the TCP protocol [can be found here]({{< relref "addons-events#tcp-events">}}).
{{< example src="examples/addons/tcp-simple.py" lang="py" >}}
For WebSocket-related objects please look at the [tcp][] module to find
all attributes that you can use when scripting.
[tcp]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/tcp.py

11
docs/src/content/api/mitmproxy.addonmanager.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.addonmanager"
url: "api/mitmproxy/addonmanager.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/addonmanager.html" >}}

11
docs/src/content/api/mitmproxy.certs.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.certs"
url: "api/mitmproxy/certs.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/certs.html" >}}

11
docs/src/content/api/mitmproxy.connection.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.connection"
url: "api/mitmproxy/connection.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/connection.html" >}}

11
docs/src/content/api/mitmproxy.coretypes.multidict.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.coretypes.multidict"
url: "api/mitmproxy/coretypes/multidict.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/coretypes/multidict.html" >}}

11
docs/src/content/api/mitmproxy.flow.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.flow"
url: "api/mitmproxy/flow.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/flow.html" >}}

11
docs/src/content/api/mitmproxy.http.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.http"
url: "api/mitmproxy/http.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/http.html" >}}

11
docs/src/content/api/mitmproxy.net.server_spec.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.net.server_spec"
url: "api/mitmproxy/net/server_spec.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/net/server_spec.html" >}}

11
docs/src/content/api/mitmproxy.proxy.server_hooks.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.proxy.server_hooks"
url: "api/mitmproxy/proxy/server_hooks.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/proxy/server_hooks.html" >}}

11
docs/src/content/api/mitmproxy.tcp.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.tcp"
url: "api/mitmproxy/tcp.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/tcp.html" >}}

11
docs/src/content/api/mitmproxy.websocket.md Normal file
View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.websocket"
url: "api/mitmproxy/websocket.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/websocket.html" >}}

3
docs/src/layouts/partials/edit-on-github.html
View File

@ -1,4 +1,4 @@
{{ if and .IsPage (not (getenv "DOCS_ARCHIVE")) }} {{ if and .IsPage (ne .Type "api") (not (getenv "DOCS_ARCHIVE")) }}
<a class="button is-small is-outlined is-link is-pulled-right" <a class="button is-small is-outlined is-link is-pulled-right"
target="_blank" target="_blank"
href="https://github.com/mitmproxy/mitmproxy/blob/master/docs/src/content/{{ .File.Path }}" href="https://github.com/mitmproxy/mitmproxy/blob/master/docs/src/content/{{ .File.Path }}"
@ -6,4 +6,3 @@
Edit on GitHub Edit on GitHub
</a> </a>
{{ end }} {{ end }}

16
docs/src/layouts/partials/sidemenu.html
View File

@ -3,9 +3,19 @@
{{ $currentPage := .ctx }} {{ $currentPage := .ctx }}
{{ $menuname := .menuname }} {{ $menuname := .menuname }}
{{ range $menu.ByWeight }} {{ range $menu.ByWeight }}
<li > <li>
<a class="{{ if $currentPage.IsMenuCurrent $menuname . }}is-active{{ end }}" <a class="{{ if $currentPage.IsMenuCurrent $menuname . }}is-active{{ end }}"
href="{{.URL}}">{{ .Name }}</a> href="{{.URL}}">{{ .Name }}</a>
{{ if and .HasChildren (or ($currentPage.IsMenuCurrent $menuname .) ($currentPage.HasMenuCurrent $menuname .)) }}
<ul>
{{ range .Children }}
<li>
<a class="{{ if $currentPage.IsMenuCurrent $menuname . }}is-active{{ end }}"
href="{{ .URL }}">{{ .Name }}</a>
</li>
{{ end }}
</ul>
{{ end }}
</li> </li>
{{end}} {{end}}
</ul> </ul>

3
examples/addons/scripting-minimal-example.py → examples/addons/anatomy2.py
View File

@ -1,2 +1,5 @@
"""An addon using the abbreviated scripting syntax."""
def request(flow): def request(flow):
flow.request.headers["myheader"] = "value" flow.request.headers["myheader"] = "value"

1
examples/addons/http-trailers.py
View File

@ -36,7 +36,6 @@ def request(flow: http.HTTPFlow):
def response(flow: http.HTTPFlow): def response(flow: http.HTTPFlow):
assert flow.response # make type checker happy
if flow.response.trailers: if flow.response.trailers:
print("HTTP Trailers detected! Response contains:", flow.response.trailers) print("HTTP Trailers detected! Response contains:", flow.response.trailers)

14
examples/addons/internet-in-mirror.py Normal file
View File

@ -0,0 +1,14 @@
"""
Mirror all web pages.
Useful if you are living down under.
"""
from mitmproxy import http
def response(flow: http.HTTPFlow) -> None:
if flow.response and flow.response.content:
flow.response.content = flow.response.content.replace(
b"</head>",
b"<style>body {transform: scaleX(-1);}</style></head>"
)

12
examples/addons/internet_in_mirror.py
View File

@ -1,12 +0,0 @@
"""
Mirror all web pages.
Useful if you are living down under.
"""
from mitmproxy import http
def response(flow: http.HTTPFlow) -> None:
assert flow.response # make type checker happy
reflector = b"<style>body {transform: scaleX(-1);}</style></head>"
flow.response.content = flow.response.content.replace(b"</head>", reflector)

24
examples/addons/websocket-inject-message.py
View File

@ -1,24 +0,0 @@
"""
Inject a WebSocket message into a running connection.
This example shows how to inject a WebSocket message to the client.
Every new WebSocket connection will trigger a new asyncio task that
periodically injects a new message to the client.
"""
import asyncio
import mitmproxy.websocket
class InjectWebSocketMessage:
async def inject(self, flow: mitmproxy.websocket.WebSocketFlow):
i = 0
while not flow.ended and not flow.error:
await asyncio.sleep(5)
flow.inject_message(flow.client_conn, f'This is the #{i} injected message!')
i += 1
def websocket_start(self, flow):
asyncio.get_event_loop().create_task(self.inject(flow))
addons = [InjectWebSocketMessage()]

5
mitmproxy/addonmanager.py
View File

@ -98,6 +98,11 @@ class Loader:
) )
def add_command(self, path: str, func: typing.Callable) -> None: def add_command(self, path: str, func: typing.Callable) -> None:
"""Add a command to mitmproxy.
Unless you are generating commands programatically,
this API should be avoided. Decorate your function with `@mitmproxy.command.command` instead.
"""
self.master.commands.add(path, func) self.master.commands.add(path, func)

8
mitmproxy/addons/dumper.py
View File

@ -80,11 +80,11 @@ class Dumper:
def _echo_headers(self, headers: http.Headers): def _echo_headers(self, headers: http.Headers):
for k, v in headers.fields: for k, v in headers.fields:
k = strutils.bytes_to_escaped_str(k) ks = strutils.bytes_to_escaped_str(k)
v = strutils.bytes_to_escaped_str(v) vs = strutils.bytes_to_escaped_str(v)
out = "{}: {}".format( out = "{}: {}".format(
click.style(k, fg="blue"), click.style(ks, fg="blue"),
click.style(v) click.style(vs)
) )
self.echo(out, ident=4) self.echo(out, ident=4)

1
mitmproxy/certs.py
View File

@ -39,6 +39,7 @@ rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI=
class Cert(serializable.Serializable): class Cert(serializable.Serializable):
"""Representation of a (TLS) certificate."""
_cert: x509.Certificate _cert: x509.Certificate
def __init__(self, cert: x509.Certificate): def __init__(self, cert: x509.Certificate):

8
mitmproxy/connection.py
View File

@ -29,7 +29,7 @@ class Connection(serializable.Serializable, metaclass=ABCMeta):
Base class for client and server connections. Base class for client and server connections.
The connection object only exposes metadata about the connection, but not the underlying socket object. The connection object only exposes metadata about the connection, but not the underlying socket object.
This is intentional, all I/O should be handled by mitmproxy.proxy.server exclusively. This is intentional, all I/O should be handled by `mitmproxy.proxy.server` exclusively.
""" """
# all connections have a unique id. While # all connections have a unique id. While
# f.client_conn == f2.client_conn already holds true for live flows (where we have object identity), # f.client_conn == f2.client_conn already holds true for live flows (where we have object identity),
@ -92,12 +92,12 @@ class Connection(serializable.Serializable, metaclass=ABCMeta):
@property @property
def connected(self) -> bool: def connected(self) -> bool:
"""`True` if Connection.state is ConnectionState.OPEN, `False` otherwise. Read-only.""" """*Read-only:* `True` if Connection.state is ConnectionState.OPEN, `False` otherwise."""
return self.state is ConnectionState.OPEN return self.state is ConnectionState.OPEN
@property @property
def tls_established(self) -> bool: def tls_established(self) -> bool:
"""`True` if TLS has been established, `False` otherwise. Read-only.""" """*Read-only:* `True` if TLS has been established, `False` otherwise."""
return self.timestamp_tls_setup is not None return self.timestamp_tls_setup is not None
def __eq__(self, other): def __eq__(self, other):
@ -143,7 +143,7 @@ class Client(Connection):
timestamp_start: float timestamp_start: float
"""*Timestamp:* TCP SYN received""" """*Timestamp:* TCP SYN received"""
def __init__(self, peername, sockname, timestamp_start): def __init__(self, peername: Address, sockname: Address, timestamp_start: float):
self.id = str(uuid.uuid4()) self.id = str(uuid.uuid4())
self.peername = peername self.peername = peername
self.sockname = sockname self.sockname = sockname

83
mitmproxy/coretypes/multidict.py
View File

@ -1,10 +1,26 @@
from abc import ABCMeta, abstractmethod from abc import ABCMeta
from abc import abstractmethod
from typing import Iterator
from typing import List
from typing import MutableMapping
from typing import Sequence
from typing import Tuple
from typing import TypeVar
from collections.abc import MutableMapping
from mitmproxy.coretypes import serializable from mitmproxy.coretypes import serializable
KT = TypeVar('KT')
VT = TypeVar('VT')
class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta):
"""
A MultiDict is a dictionary-like data structure that supports multiple values per key.
"""
fields: Tuple[Tuple[KT, VT], ...]
"""The underlying raw datastructure."""
class _MultiDict(MutableMapping, metaclass=ABCMeta):
def __repr__(self): def __repr__(self):
fields = ( fields = (
repr(field) repr(field)
@ -17,7 +33,7 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def _reduce_values(values): def _reduce_values(values: Sequence[VT]) -> VT:
""" """
If a user accesses multidict["foo"], this method If a user accesses multidict["foo"], this method
reduces all values for "foo" to a single value that is returned. reduces all values for "foo" to a single value that is returned.
@ -27,22 +43,22 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def _kconv(key): def _kconv(key: KT) -> KT:
""" """
This method converts a key to its canonical representation. This method converts a key to its canonical representation.
For example, HTTP headers are case-insensitive, so this method returns key.lower(). For example, HTTP headers are case-insensitive, so this method returns key.lower().
""" """
def __getitem__(self, key): def __getitem__(self, key: KT) -> VT:
values = self.get_all(key) values = self.get_all(key)
if not values: if not values:
raise KeyError(key) raise KeyError(key)
return self._reduce_values(values) return self._reduce_values(values)
def __setitem__(self, key, value): def __setitem__(self, key: KT, value: VT) -> None:
self.set_all(key, [value]) self.set_all(key, [value])
def __delitem__(self, key): def __delitem__(self, key: KT) -> None:
if key not in self: if key not in self:
raise KeyError(key) raise KeyError(key)
key = self._kconv(key) key = self._kconv(key)
@ -51,7 +67,7 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
if key != self._kconv(field[0]) if key != self._kconv(field[0])
) )
def __iter__(self): def __iter__(self) -> Iterator[KT]:
seen = set() seen = set()
for key, _ in self.fields: for key, _ in self.fields:
key_kconv = self._kconv(key) key_kconv = self._kconv(key)
@ -59,15 +75,15 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
seen.add(key_kconv) seen.add(key_kconv)
yield key yield key
def __len__(self): def __len__(self) -> int:
return len({self._kconv(key) for key, _ in self.fields}) return len({self._kconv(key) for key, _ in self.fields})
def __eq__(self, other): def __eq__(self, other) -> bool:
if isinstance(other, MultiDict): if isinstance(other, MultiDict):
return self.fields == other.fields return self.fields == other.fields
return False return False
def get_all(self, key): def get_all(self, key: KT) -> List[VT]:
""" """
Return the list of all values for a given key. Return the list of all values for a given key.
If that key is not in the MultiDict, the return value will be an empty list. If that key is not in the MultiDict, the return value will be an empty list.
@ -79,13 +95,13 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
if self._kconv(k) == key if self._kconv(k) == key
] ]
def set_all(self, key, values): def set_all(self, key: KT, values: List[VT]) -> None:
""" """
Remove the old values for a key and add new ones. Remove the old values for a key and add new ones.
""" """
key_kconv = self._kconv(key) key_kconv = self._kconv(key)
new_fields = [] new_fields: List[Tuple[KT, VT]] = []
for field in self.fields: for field in self.fields:
if self._kconv(field[0]) == key_kconv: if self._kconv(field[0]) == key_kconv:
if values: if values:
@ -100,55 +116,49 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
) )
self.fields = tuple(new_fields) self.fields = tuple(new_fields)
def add(self, key, value): def add(self, key: KT, value: VT) -> None:
""" """
Add an additional value for the given key at the bottom. Add an additional value for the given key at the bottom.
""" """
self.insert(len(self.fields), key, value) self.insert(len(self.fields), key, value)
def insert(self, index, key, value): def insert(self, index: int, key: KT, value: VT) -> None:
""" """
Insert an additional value for the given key at the specified position. Insert an additional value for the given key at the specified position.
""" """
item = (key, value) item = (key, value)
self.fields = self.fields[:index] + (item,) + self.fields[index:] self.fields = self.fields[:index] + (item,) + self.fields[index:]
def keys(self, multi=False): def keys(self, multi: bool = False):
""" """
Get all keys. Get all keys.
Args: If `multi` is True, one key per value will be returned.
multi(bool): If `multi` is False, duplicate keys will only be returned once.
If True, one key per value will be returned.
If False, duplicate keys will only be returned once.
""" """
return ( return (
k k
for k, _ in self.items(multi) for k, _ in self.items(multi)
) )
def values(self, multi=False): def values(self, multi: bool = False):
""" """
Get all values. Get all values.
Args: If `multi` is True, all values will be returned.
multi(bool): If `multi` is False, only the first value per key will be returned.
If True, all values will be returned.
If False, only the first value per key will be returned.
""" """
return ( return (
v v
for _, v in self.items(multi) for _, v in self.items(multi)
) )
def items(self, multi=False): def items(self, multi: bool = False):
""" """
Get all (key, value) tuples. Get all (key, value) tuples.
Args: If `multi` is True, all `(key, value)` pairs will be returned.
multi(bool): If False, only one tuple per key is returned.
If True, all (key, value) pairs will be returned
If False, only the first (key, value) pair per unique key will be returned.
""" """
if multi: if multi:
return self.fields return self.fields
@ -156,7 +166,9 @@ class _MultiDict(MutableMapping, metaclass=ABCMeta):
return super().items() return super().items()
class MultiDict(_MultiDict, serializable.Serializable): class MultiDict(_MultiDict[KT, VT], serializable.Serializable):
"""A concrete MultiDict, storing its own data."""
def __init__(self, fields=()): def __init__(self, fields=()):
super().__init__() super().__init__()
self.fields = tuple( self.fields = tuple(
@ -182,12 +194,13 @@ class MultiDict(_MultiDict, serializable.Serializable):
return cls(state) return cls(state)
class MultiDictView(_MultiDict): class MultiDictView(_MultiDict[KT, VT]):
""" """
The MultiDictView provides the MultiDict interface over calculated data. The MultiDictView provides the MultiDict interface over calculated data.
The view itself contains no state - data is retrieved from the parent on The view itself contains no state - data is retrieved from the parent on
request, and stored back to the parent on change. request, and stored back to the parent on change.
""" """
def __init__(self, getter, setter): def __init__(self, getter, setter):
self._getter = getter self._getter = getter
self._setter = setter self._setter = setter
@ -204,7 +217,7 @@ class MultiDictView(_MultiDict):
# multiple elements exist with the same key. # multiple elements exist with the same key.
return values[0] return values[0]
@property @property # type: ignore
def fields(self): def fields(self):
return self._getter() return self._getter()
@ -212,5 +225,5 @@ class MultiDictView(_MultiDict):
def fields(self, value): def fields(self, value):
self._setter(value) self._setter(value)
def copy(self): def copy(self) -> "MultiDict[KT,VT]":
return MultiDict(self.fields) return MultiDict(self.fields)

84
mitmproxy/flow.py
View File

@ -10,21 +10,21 @@ from mitmproxy import version
class Error(stateobject.StateObject): class Error(stateobject.StateObject):
""" """
An Error. An Error.
This is distinct from an protocol error response (say, a HTTP code 500), This is distinct from an protocol error response (say, a HTTP code 500),
which is represented by a normal `mitmproxy.http.Response` object. This class is which is represented by a normal `mitmproxy.http.Response` object. This class is
responsible for indicating errors that fall outside of normal protocol responsible for indicating errors that fall outside of normal protocol
communications, like interrupted connections, timeouts, protocol errors. communications, like interrupted connections, timeouts, or protocol errors.
""" """
msg: str msg: str
"""Message describing the error.""" """Message describing the error."""
timestamp: float timestamp: float
"""Unix timestamp""" """Unix timestamp of when this error happened."""
KILLED_MESSAGE = "Connection killed." KILLED_MESSAGE: typing.ClassVar[str] = "Connection killed."
def __init__(self, msg: str, timestamp: typing.Optional[float] = None) -> None: def __init__(self, msg: str, timestamp: typing.Optional[float] = None) -> None:
"""Create an error. If no timestamp is passed, the current time is used.""" """Create an error. If no timestamp is passed, the current time is used."""
@ -53,8 +53,46 @@ class Error(stateobject.StateObject):
class Flow(stateobject.StateObject): class Flow(stateobject.StateObject):
""" """
A Flow is a collection of objects representing a single transaction. Base class for network flows. A flow is a collection of objects,
This class is usually subclassed for each protocol, e.g. HTTPFlow. for example HTTP request/response pairs or a list of TCP messages.
See also:
- mitmproxy.http.HTTPFlow
- mitmproxy.tcp.TCPFlow
"""
client_conn: connection.Client
"""The client that connected to mitmproxy."""
server_conn: connection.Server
"""
The server mitmproxy connected to.
Some flows may never cause mitmproxy to initiate a server connection,
for example because their response is replayed by mitmproxy itself.
To simplify implementation, those flows will still have a `server_conn` attribute
with a `timestamp_start` set to `None`.
"""
error: typing.Optional[Error] = None
"""A connection or protocol error affecting this flow."""
intercepted: bool
"""
If `True`, the flow is currently paused by mitmproxy.
We're waiting for a user action to forward the flow to its destination.
"""
marked: bool
"""
If `True`, this flow has been marked by the user.
"""
is_replay: typing.Optional[str]
"""
This attribute indicates if this flow has been replayed in either direction.
- a value of `request` indicates that the request has been artifically replayed by mitmproxy to the server.
- a value of `response` indicates that the response to the client's request has been set by server replay.
""" """
def __init__( def __init__(
@ -70,7 +108,6 @@ class Flow(stateobject.StateObject):
self.server_conn = server_conn self.server_conn = server_conn
self.live = live self.live = live
self.error: typing.Optional[Error] = None
self.intercepted: bool = False self.intercepted: bool = False
self._backup: typing.Optional[Flow] = None self._backup: typing.Optional[Flow] = None
self.reply: typing.Optional[controller.Reply] = None self.reply: typing.Optional[controller.Reply] = None
@ -111,6 +148,7 @@ class Flow(stateobject.StateObject):
return f return f
def copy(self): def copy(self):
"""Make a copy of this flow."""
f = super().copy() f = super().copy()
f.live = False f.live = False
if self.reply is not None: if self.reply is not None:
@ -119,7 +157,7 @@ class Flow(stateobject.StateObject):
def modified(self): def modified(self):
""" """
Has this Flow been modified? `True` if this file has been modified by a user, `False` otherwise.
""" """
if self._backup: if self._backup:
return self._backup != self.get_state() return self._backup != self.get_state()
@ -128,8 +166,7 @@ class Flow(stateobject.StateObject):
def backup(self, force=False): def backup(self, force=False):
""" """
Save a backup of this Flow, which can be reverted to using a Save a backup of this flow, which can be restored by calling `Flow.revert()`.
call to .revert().
""" """
if not self._backup: if not self._backup:
self._backup = self.get_state() self._backup = self.get_state()
@ -144,6 +181,7 @@ class Flow(stateobject.StateObject):
@property @property
def killable(self): def killable(self):
"""*Read-only:* `True` if this flow can be killed, `False` otherwise."""
return ( return (
self.reply and self.reply and
self.reply.state in {"start", "taken"} and self.reply.state in {"start", "taken"} and
@ -152,7 +190,7 @@ class Flow(stateobject.StateObject):
def kill(self): def kill(self):
""" """
Kill this request. Kill this flow. The current request/response will not be forwarded to its destination.
""" """
if not self.killable: if not self.killable:
raise exceptions.ControlException("Flow is not killable.") raise exceptions.ControlException("Flow is not killable.")
@ -162,8 +200,8 @@ class Flow(stateobject.StateObject):
def intercept(self): def intercept(self):
""" """
Intercept this Flow. Processing will stop until resume is Intercept this Flow. Processing will stop until resume is
called. called.
""" """
if self.intercepted: if self.intercepted:
return return
@ -172,7 +210,7 @@ class Flow(stateobject.StateObject):
def resume(self): def resume(self):
""" """
Continue with the flow - called after an intercept(). Continue with the flow – called after an intercept().
""" """
if not self.intercepted: if not self.intercepted:
return return
@ -183,5 +221,15 @@ class Flow(stateobject.StateObject):
@property @property
def timestamp_start(self) -> float: def timestamp_start(self) -> float:
"""Start time of the flow.""" """
*Read-only:* Start time of the flow.
Depending on the flow type, this property is an alias for
`mitmproxy.connection.Client.timestamp_start` or `mitmproxy.http.Request.timestamp_start`.
"""
return self.client_conn.timestamp_start return self.client_conn.timestamp_start
__all__ = [
"Flow",
"Error",
]

438
mitmproxy/http.py
View File

@ -3,23 +3,30 @@ import time
import urllib.parse import urllib.parse
from dataclasses import dataclass from dataclasses import dataclass
from dataclasses import fields from dataclasses import fields
from email.utils import formatdate, mktime_tz, parsedate_tz from email.utils import formatdate
from typing import Callable, cast from email.utils import mktime_tz
from email.utils import parsedate_tz
from typing import Callable
from typing import Dict from typing import Dict
from typing import Iterable from typing import Iterable
from typing import Iterator
from typing import List
from typing import Mapping from typing import Mapping
from typing import Optional from typing import Optional
from typing import Tuple from typing import Tuple
from typing import Union from typing import Union
from typing import cast
from mitmproxy import flow, connection from mitmproxy import flow
from mitmproxy.coretypes import multidict from mitmproxy.coretypes import multidict
from mitmproxy.coretypes import serializable from mitmproxy.coretypes import serializable
from mitmproxy.net import encoding from mitmproxy.net import encoding
from mitmproxy.net.http import cookies, multipart from mitmproxy.net.http import cookies
from mitmproxy.net.http import multipart
from mitmproxy.net.http import status_codes from mitmproxy.net.http import status_codes
from mitmproxy.net.http import url from mitmproxy.net.http import url
from mitmproxy.net.http.headers import assemble_content_type, parse_content_type from mitmproxy.net.http.headers import assemble_content_type
from mitmproxy.net.http.headers import parse_content_type
from mitmproxy.utils import human from mitmproxy.utils import human
from mitmproxy.utils import strutils from mitmproxy.utils import strutils
from mitmproxy.utils import typecheck from mitmproxy.utils import typecheck
@ -28,72 +35,70 @@ from mitmproxy.utils.strutils import always_str
# While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded. # While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded.
def _native(x): def _native(x: bytes) -> str:
return x.decode("utf-8", "surrogateescape") return x.decode("utf-8", "surrogateescape")
def _always_bytes(x): def _always_bytes(x: Union[str, bytes]) -> bytes:
return strutils.always_bytes(x, "utf-8", "surrogateescape") return strutils.always_bytes(x, "utf-8", "surrogateescape")
class Headers(multidict.MultiDict): # This cannot be easily typed with mypy yet, so we just specify MultiDict without concrete types.
class Headers(multidict.MultiDict): # type: ignore
""" """
Header class which allows both convenient access to individual headers as well as Header class which allows both convenient access to individual headers as well as
direct access to the underlying raw data. Provides a full dictionary interface. direct access to the underlying raw data. Provides a full dictionary interface.
Example: Create headers with keyword arguments:
>>> h = Headers(host="example.com", content_type="application/xml")
.. code-block:: python Headers mostly behave like a normal dict:
>>> h["Host"]
"example.com"
# Create headers with keyword arguments Headers are case insensitive:
>>> h = Headers(host="example.com", content_type="application/xml") >>> h["host"]
"example.com"
# Headers mostly behave like a normal dict. Headers can also be created from a list of raw (header_name, header_value) byte tuples:
>>> h["Host"] >>> h = Headers([
"example.com" (b"Host",b"example.com"),
(b"Accept",b"text/html"),
(b"accept",b"application/xml")
])
# HTTP Headers are case insensitive Multiple headers are folded into a single header as per RFC 7230:
>>> h["host"] >>> h["Accept"]
"example.com" "text/html, application/xml"
# Headers can also be created from a list of raw (header_name, header_value) byte tuples Setting a header removes all existing headers with the same name:
>>> h = Headers([ >>> h["Accept"] = "application/text"
(b"Host",b"example.com"), >>> h["Accept"]
(b"Accept",b"text/html"), "application/text"
(b"accept",b"application/xml")
])
# Multiple headers are folded into a single header as per RFC7230 `bytes(h)` returns an HTTP/1 header block:
>>> h["Accept"] >>> print(bytes(h))
"text/html, application/xml" Host: example.com
Accept: application/text
# Setting a header removes all existing headers with the same name. For full control, the raw header fields can be accessed:
>>> h["Accept"] = "application/text" >>> h.fields
>>> h["Accept"]
"application/text"
# bytes(h) returns a HTTP1 header block.
>>> print(bytes(h))
Host: example.com
Accept: application/text
# For full control, the raw header fields can be accessed
>>> h.fields
Caveats: Caveats:
For use with the "Set-Cookie" header, see :py:meth:`get_all`. - For use with the "Set-Cookie" header, either use `Response.cookies` or see `Headers.get_all`.
""" """
def __init__(self, fields=(), **headers): def __init__(self, fields: Iterable[Tuple[bytes, bytes]] = (), **headers):
""" """
Args: *Args:*
fields: (optional) list of ``(name, value)`` header byte tuples, - *fields:* (optional) list of ``(name, value)`` header byte tuples,
e.g. ``[(b"Host", b"example.com")]``. All names and values must be bytes. e.g. ``[(b"Host", b"example.com")]``. All names and values must be bytes.
**headers: Additional headers to set. Will overwrite existing values from `fields`. - *\*\*headers:* Additional headers to set. Will overwrite existing values from `fields`.
For convenience, underscores in header names will be transformed to dashes - For convenience, underscores in header names will be transformed to dashes -
this behaviour does not extend to other methods. this behaviour does not extend to other methods.
If ``**headers`` contains multiple keys that have equal ``.lower()`` s,
the behavior is undefined. If ``**headers`` contains multiple keys that have equal ``.lower()`` representations,
the behavior is undefined.
""" """
super().__init__(fields) super().__init__(fields)
@ -102,41 +107,43 @@ class Headers(multidict.MultiDict):
raise TypeError("Header fields must be bytes.") raise TypeError("Header fields must be bytes.")
# content_type -> content-type # content_type -> content-type
headers = { self.update({
_always_bytes(name).replace(b"_", b"-"): _always_bytes(value) _always_bytes(name).replace(b"_", b"-"): _always_bytes(value)
for name, value in headers.items() for name, value in headers.items()
} })
self.update(headers)
fields: Tuple[Tuple[bytes, bytes], ...]
@staticmethod @staticmethod
def _reduce_values(values): def _reduce_values(values) -> str:
# Headers can be folded # Headers can be folded
return ", ".join(values) return ", ".join(values)
@staticmethod @staticmethod
def _kconv(key): def _kconv(key) -> str:
# Headers are case-insensitive # Headers are case-insensitive
return key.lower() return key.lower()
def __bytes__(self): def __bytes__(self) -> bytes:
if self.fields: if self.fields:
return b"\r\n".join(b": ".join(field) for field in self.fields) + b"\r\n" return b"\r\n".join(b": ".join(field) for field in self.fields) + b"\r\n"
else: else:
return b"" return b""
def __delitem__(self, key): def __delitem__(self, key: Union[str, bytes]) -> None:
key = _always_bytes(key) key = _always_bytes(key)
super().__delitem__(key) super().__delitem__(key)
def __iter__(self): def __iter__(self) -> Iterator[str]:
for x in super().__iter__(): for x in super().__iter__():
yield _native(x) yield _native(x)
def get_all(self, name): def get_all(self, name: Union[str, bytes]) -> List[str]:
""" """
Like :py:meth:`get`, but does not fold multiple headers into a single one. Like `Headers.get`, but does not fold multiple headers into a single one.
This is useful for Set-Cookie headers, which do not support folding. This is useful for Set-Cookie headers, which do not support folding.
See also: https://tools.ietf.org/html/rfc7230#section-3.2.2
*See also:* <https://tools.ietf.org/html/rfc7230#section-3.2.2>
""" """
name = _always_bytes(name) name = _always_bytes(name)
return [ return [
@ -144,16 +151,16 @@ class Headers(multidict.MultiDict):
super().get_all(name) super().get_all(name)
] ]
def set_all(self, name, values): def set_all(self, name: Union[str, bytes], values: List[Union[str, bytes]]):
""" """
Explicitly set multiple headers for the given key. Explicitly set multiple headers for the given key.
See: :py:meth:`get_all` See `Headers.get_all`.
""" """
name = _always_bytes(name) name = _always_bytes(name)
values = [_always_bytes(x) for x in values] values = [_always_bytes(x) for x in values]
return super().set_all(name, values) return super().set_all(name, values)
def insert(self, index, key, value): def insert(self, index: int, key: Union[str, bytes], value: Union[str, bytes]):
key = _always_bytes(key) key = _always_bytes(key)
value = _always_bytes(value) value = _always_bytes(value)
super().insert(index, key, value) super().insert(index, key, value)
@ -222,6 +229,8 @@ class ResponseData(MessageData):
class Message(serializable.Serializable): class Message(serializable.Serializable):
"""Base class for `Request` and `Response`."""
@classmethod @classmethod
def from_state(cls, state): def from_state(cls, state):
return cls(**state) return cls(**state)
@ -233,12 +242,21 @@ class Message(serializable.Serializable):
self.data.set_state(state) self.data.set_state(state)
data: MessageData data: MessageData
stream: Union[Callable, bool] = False stream: Union[Callable[[bytes], bytes], bool] = False
"""
If `True`, the message body will not be buffered on the proxy
but immediately streamed to the destination instead.
Alternatively, a transformation function can be specified, but please note
that packet should not be relied upon.
This attribute must be set in the `requestheaders` or `responseheaders` hook.
Setting it in `request` or `response` is already too late, mitmproxy has buffered the message body already.
"""
@property @property
def http_version(self) -> str: def http_version(self) -> str:
""" """
Version string, e.g. "HTTP/1.1" HTTP version string, for example `HTTP/1.1`.
""" """
return self.data.http_version.decode("utf-8", "surrogateescape") return self.data.http_version.decode("utf-8", "surrogateescape")
@ -272,7 +290,7 @@ class Message(serializable.Serializable):
@property @property
def trailers(self) -> Optional[Headers]: def trailers(self) -> Optional[Headers]:
""" """
The HTTP trailers. The [HTTP trailers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Trailer).
""" """
return self.data.trailers return self.data.trailers
@ -283,9 +301,11 @@ class Message(serializable.Serializable):
@property @property
def raw_content(self) -> Optional[bytes]: def raw_content(self) -> Optional[bytes]:
""" """
The raw (potentially compressed) HTTP message body as bytes. The raw (potentially compressed) HTTP message body.
See also: :py:attr:`content`, :py:class:`text` In contrast to `Message.content` and `Message.text`, accessing this property never raises.
*See also:* `Message.content`, `Message.text`
""" """
return self.data.content return self.data.content
@ -293,31 +313,35 @@ class Message(serializable.Serializable):
def raw_content(self, content: Optional[bytes]) -> None: def raw_content(self, content: Optional[bytes]) -> None:
self.data.content = content self.data.content = content
def get_content(self, strict: bool = True) -> Optional[bytes]: @property
def content(self) -> Optional[bytes]:
""" """
The uncompressed HTTP message body as bytes. The uncompressed HTTP message body as bytes.
Raises: Accessing this attribute may raise a `ValueError` when the HTTP content-encoding is invalid.
ValueError, when the HTTP content-encoding is invalid and strict is True.
See also: :py:class:`raw_content`, :py:attr:`text` *See also:* `Message.raw_content`, `Message.text`
""" """
if self.raw_content is None: return self.get_content()
return None
ce = self.headers.get("content-encoding") @content.setter
if ce: def content(self, value: Optional[bytes]) -> None:
try: self.set_content(value)
content = encoding.decode(self.raw_content, ce)
# A client may illegally specify a byte -> str encoding here (e.g. utf8) @property
if isinstance(content, str): def text(self) -> Optional[str]:
raise ValueError(f"Invalid Content-Encoding: {ce}") """
return content The uncompressed and decoded HTTP message body as text.
except ValueError:
if strict: Accessing this attribute may raise a `ValueError` when either content-encoding or charset is invalid.
raise
return self.raw_content *See also:* `Message.raw_content`, `Message.content`
else: """
return self.raw_content return self.get_text()
@text.setter
def text(self, value: Optional[str]) -> None:
self.set_text(value)
def set_content(self, value: Optional[bytes]) -> None: def set_content(self, value: Optional[bytes]) -> None:
if value is None: if value is None:
@ -338,29 +362,27 @@ class Message(serializable.Serializable):
self.raw_content = value self.raw_content = value
self.headers["content-length"] = str(len(self.raw_content)) self.headers["content-length"] = str(len(self.raw_content))
content = property(get_content, set_content) def get_content(self, strict: bool = True) -> Optional[bytes]:
@property
def timestamp_start(self) -> float:
""" """
First byte timestamp Similar to `Message.content`, but does not raise if `strict` is `False`.
Instead, the compressed message body is returned as-is.
""" """
return self.data.timestamp_start if self.raw_content is None:
return None
@timestamp_start.setter ce = self.headers.get("content-encoding")
def timestamp_start(self, timestamp_start: float) -> None: if ce:
self.data.timestamp_start = timestamp_start try:
content = encoding.decode(self.raw_content, ce)
@property # A client may illegally specify a byte -> str encoding here (e.g. utf8)
def timestamp_end(self) -> Optional[float]: if isinstance(content, str):
""" raise ValueError(f"Invalid Content-Encoding: {ce}")
Last byte timestamp return content
""" except ValueError:
return self.data.timestamp_end if strict:
raise
@timestamp_end.setter return self.raw_content
def timestamp_end(self, timestamp_end: Optional[float]): else:
self.data.timestamp_end = timestamp_end return self.raw_content
def _get_content_type_charset(self) -> Optional[str]: def _get_content_type_charset(self) -> Optional[str]:
ct = parse_content_type(self.headers.get("content-type", "")) ct = parse_content_type(self.headers.get("content-type", ""))
@ -391,14 +413,26 @@ class Message(serializable.Serializable):
return enc return enc
def set_text(self, text: Optional[str]) -> None:
if text is None:
self.content = None
return
enc = self._guess_encoding()
try:
self.content = cast(bytes, encoding.encode(text, enc))
except ValueError:
# Fall back to UTF-8 and update the content-type header.
ct = parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {})
ct[2]["charset"] = "utf-8"
self.headers["content-type"] = assemble_content_type(*ct)
enc = "utf8"
self.content = text.encode(enc, "surrogateescape")
def get_text(self, strict: bool = True) -> Optional[str]: def get_text(self, strict: bool = True) -> Optional[str]:
""" """
The uncompressed and decoded HTTP message body as text. Similar to `Message.text`, but does not raise if `strict` is `False`.
Instead, the message body is returned as surrogate-escaped UTF-8.
Raises:
ValueError, when either content-encoding or charset is invalid and strict is True.
See also: :py:attr:`content`, :py:class:`raw_content`
""" """
content = self.get_content(strict) content = self.get_content(strict)
if content is None: if content is None:
@ -411,23 +445,27 @@ class Message(serializable.Serializable):
raise raise
return content.decode("utf8", "surrogateescape") return content.decode("utf8", "surrogateescape")
def set_text(self, text: Optional[str]) -> None: @property
if text is None: def timestamp_start(self) -> float:
self.content = None """
return *Timestamp:* Headers received.
enc = self._guess_encoding() """
return self.data.timestamp_start
try: @timestamp_start.setter
self.content = encoding.encode(text, enc) def timestamp_start(self, timestamp_start: float) -> None:
except ValueError: self.data.timestamp_start = timestamp_start
# Fall back to UTF-8 and update the content-type header.
ct = parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {})
ct[2]["charset"] = "utf-8"
self.headers["content-type"] = assemble_content_type(*ct)
enc = "utf8"
self.content = text.encode(enc, "surrogateescape")
text = property(get_text, set_text) @property
def timestamp_end(self) -> Optional[float]:
"""
*Timestamp:* Last byte received.
"""
return self.data.timestamp_end
@timestamp_end.setter
def timestamp_end(self, timestamp_end: Optional[float]):
self.data.timestamp_end = timestamp_end
def decode(self, strict: bool = True) -> None: def decode(self, strict: bool = True) -> None:
""" """
@ -435,26 +473,25 @@ class Message(serializable.Serializable):
removes the header. If there is no Content-Encoding header, no removes the header. If there is no Content-Encoding header, no
action is taken. action is taken.
Raises: *Raises:*
ValueError, when the content-encoding is invalid and strict is True. - `ValueError`, when the content-encoding is invalid and strict is True.
""" """
decoded = self.get_content(strict) decoded = self.get_content(strict)
self.headers.pop("content-encoding", None) self.headers.pop("content-encoding", None)
self.content = decoded self.content = decoded
def encode(self, e: str) -> None: def encode(self, encoding: str) -> None:
""" """
Encodes body with the encoding e, where e is "gzip", "deflate", "identity", "br", or "zstd". Encodes body with the given encoding, where e is "gzip", "deflate", "identity", "br", or "zstd".
Any existing content-encodings are overwritten, Any existing content-encodings are overwritten, the content is not decoded beforehand.
the content is not decoded beforehand.
Raises: *Raises:*
ValueError, when the specified content-encoding is invalid. - `ValueError`, when the specified content-encoding is invalid.
""" """
self.headers["content-encoding"] = e self.headers["content-encoding"] = encoding
self.content = self.raw_content self.content = self.raw_content
if "content-encoding" not in self.headers: if "content-encoding" not in self.headers:
raise ValueError("Invalid content encoding {}".format(repr(e))) raise ValueError("Invalid content encoding {}".format(repr(encoding)))
class Request(Message): class Request(Message):
@ -474,7 +511,7 @@ class Request(Message):
http_version: bytes, http_version: bytes,
headers: Union[Headers, Tuple[Tuple[bytes, bytes], ...]], headers: Union[Headers, Tuple[Tuple[bytes, bytes], ...]],
content: Optional[bytes], content: Optional[bytes],
trailers: Union[None, Headers, Tuple[Tuple[bytes, bytes], ...]], trailers: Union[Headers, Tuple[Tuple[bytes, bytes], ...], None],
timestamp_start: float, timestamp_start: float,
timestamp_end: Optional[float], timestamp_end: Optional[float],
): ):
@ -543,7 +580,7 @@ class Request(Message):
for k, v in headers.items() for k, v in headers.items()
) )
elif isinstance(headers, Iterable): elif isinstance(headers, Iterable):
headers = Headers(headers) headers = Headers(headers) # type: ignore
else: else:
raise TypeError("Expected headers to be an iterable or dict, but is {}.".format( raise TypeError("Expected headers to be an iterable or dict, but is {}.".format(
type(headers).__name__ type(headers).__name__
@ -578,7 +615,7 @@ class Request(Message):
@property @property
def first_line_format(self) -> str: def first_line_format(self) -> str:
""" """
HTTP request form as defined in `RFC7230 <https://tools.ietf.org/html/rfc7230#section-5.3>`_. *Read-only:* HTTP request form as defined in [RFC 7230](https://tools.ietf.org/html/rfc7230#section-5.3).
origin-form and asterisk-form are subsumed as "relative". origin-form and asterisk-form are subsumed as "relative".
""" """
@ -617,9 +654,12 @@ class Request(Message):
HTTP request authority. HTTP request authority.
For HTTP/1, this is the authority portion of the request target For HTTP/1, this is the authority portion of the request target
(in either absolute-form or authority-form) (in either absolute-form or authority-form).
For origin-form and asterisk-form requests, this property is set to an empty string.
For HTTP/2, this is the :authority pseudo header. For HTTP/2, this is the :authority pseudo header.
*See also:* `Request.host`, `Request.host_header`, `Request.pretty_host`
""" """
try: try:
return self.data.authority.decode("idna") return self.data.authority.decode("idna")
@ -638,11 +678,13 @@ class Request(Message):
@property @property
def host(self) -> str: def host(self) -> str:
""" """
Target host. This may be parsed from the raw request Target server for this request. This may be parsed from the raw request
(e.g. from a ``GET http://example.com/ HTTP/1.1`` request line) (e.g. from a ``GET http://example.com/ HTTP/1.1`` request line)
or inferred from the proxy mode (e.g. an IP in transparent mode). or inferred from the proxy mode (e.g. an IP in transparent mode).
Setting the host attribute also updates the host header and authority information, if present. Setting the host attribute also updates the host header and authority information, if present.
*See also:* `Request.authority`, `Request.host_header`, `Request.pretty_host`
""" """
return self.data.host return self.data.host
@ -664,6 +706,8 @@ class Request(Message):
This property maps to either ``request.headers["Host"]`` or This property maps to either ``request.headers["Host"]`` or
``request.authority``, depending on whether it's HTTP/1.x or HTTP/2.0. ``request.authority``, depending on whether it's HTTP/1.x or HTTP/2.0.
*See also:* `Request.authority`,`Request.host`, `Request.pretty_host`
""" """
if self.is_http2: if self.is_http2:
return self.authority or self.data.headers.get("Host", None) return self.authority or self.data.headers.get("Host", None)
@ -686,7 +730,7 @@ class Request(Message):
@property @property
def port(self) -> int: def port(self) -> int:
""" """
Target port Target port.
""" """
return self.data.port return self.data.port
@ -709,7 +753,9 @@ class Request(Message):
@property @property
def url(self) -> str: def url(self) -> str:
""" """
The URL string, constructed from the request's URL components. The full URL string, constructed from `Request.scheme`, `Request.host`, `Request.port` and `Request.path`.
Settings this property updates these attributes as well.
""" """
if self.first_line_format == "authority": if self.first_line_format == "authority":
return f"{self.host}:{self.port}" return f"{self.host}:{self.port}"
@ -723,9 +769,11 @@ class Request(Message):
@property @property
def pretty_host(self) -> str: def pretty_host(self) -> str:
""" """
Similar to :py:attr:`host`, but using the host/:authority header as an additional (preferred) data source. *Read-only:* Like `Request.host`, but using `Request.host_header` header as an additional (preferred) data source.
This is useful in transparent mode where :py:attr:`host` is only an IP address, This is useful in transparent mode where `Request.host` is only an IP address.
but may not reflect the actual destination as the Host header could be spoofed.
*Warning:* When working in adversarial environments, this may not reflect the actual destination
as the Host header could be spoofed.
""" """
authority = self.host_header authority = self.host_header
if authority: if authority:
@ -736,7 +784,7 @@ class Request(Message):
@property @property
def pretty_url(self) -> str: def pretty_url(self) -> str:
""" """
Like :py:attr:`url`, but using :py:attr:`pretty_host` instead of :py:attr:`host`. *Read-only:* Like `Request.url`, but using `Request.pretty_host` instead of `Request.host`.
""" """
if self.first_line_format == "authority": if self.first_line_format == "authority":
return self.authority return self.authority
@ -760,9 +808,11 @@ class Request(Message):
self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment]) self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment])
@property @property
def query(self) -> multidict.MultiDictView: def query(self) -> multidict.MultiDictView[str, str]:
""" """
The request query string as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. The request query as a mutable mapping view on the request's path.
For the most part, this behaves like a dictionary.
Modifications to the MultiDictView update `Request.path`, and vice versa.
""" """
return multidict.MultiDictView( return multidict.MultiDictView(
self._get_query, self._get_query,
@ -781,11 +831,11 @@ class Request(Message):
self.headers["cookie"] = cookies.format_cookie_header(value) self.headers["cookie"] = cookies.format_cookie_header(value)
@property @property
def cookies(self) -> multidict.MultiDictView: def cookies(self) -> multidict.MultiDictView[str, str]:
""" """
The request cookies. The request cookies.
For the most part, this behaves like a dictionary.
An empty :py:class:`~mitmproxy.net.multidict.MultiDictView` object if the cookie monster ate them all. Modifications to the MultiDictView update `Request.headers`, and vice versa.
""" """
return multidict.MultiDictView( return multidict.MultiDictView(
self._get_cookies, self._get_cookies,
@ -797,7 +847,7 @@ class Request(Message):
self._set_cookies(value) self._set_cookies(value)
@property @property
def path_components(self): def path_components(self) -> Tuple[str, ...]:
""" """
The URL's path components as a tuple of strings. The URL's path components as a tuple of strings.
Components are unquoted. Components are unquoted.
@ -809,7 +859,7 @@ class Request(Message):
return tuple(url.unquote(i) for i in path.split("/") if i) return tuple(url.unquote(i) for i in path.split("/") if i)
@path_components.setter @path_components.setter
def path_components(self, components): def path_components(self, components: Iterable[str]):
components = map(lambda x: url.quote(x, safe=""), components) components = map(lambda x: url.quote(x, safe=""), components)
path = "/" + "/".join(components) path = "/" + "/".join(components)
_, _, _, params, query, fragment = urllib.parse.urlparse(self.url) _, _, _, params, query, fragment = urllib.parse.urlparse(self.url)
@ -817,27 +867,24 @@ class Request(Message):
def anticache(self) -> None: def anticache(self) -> None:
""" """
Modifies this request to remove headers that might produce a cached Modifies this request to remove headers that might produce a cached response.
response. That is, we remove ETags and If-Modified-Since headers.
""" """
delheaders = [ delheaders = (
"if-modified-since", "if-modified-since",
"if-none-match", "if-none-match",
] )
for i in delheaders: for i in delheaders:
self.headers.pop(i, None) self.headers.pop(i, None)
def anticomp(self) -> None: def anticomp(self) -> None:
""" """
Modifies this request to remove headers that will compress the Modify the Accept-Encoding header to only accept uncompressed responses.
resource's data.
""" """
self.headers["accept-encoding"] = "identity" self.headers["accept-encoding"] = "identity"
def constrain_encoding(self) -> None: def constrain_encoding(self) -> None:
""" """
Limits the permissible Accept-Encoding values, based on what we can Limits the permissible Accept-Encoding values, based on what we can decode appropriately.
decode appropriately.
""" """
accept_encoding = self.headers.get("accept-encoding") accept_encoding = self.headers.get("accept-encoding")
if accept_encoding: if accept_encoding:
@ -864,13 +911,14 @@ class Request(Message):
self.content = url.encode(form_data, self.get_text(strict=False)).encode() self.content = url.encode(form_data, self.get_text(strict=False)).encode()
@property @property
def urlencoded_form(self): def urlencoded_form(self) -> multidict.MultiDictView[str, str]:
""" """
The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. The URL-encoded form data.
An empty multidict.MultiDictView if the content-type indicates non-form data
or the content could not be parsed.
Starting with mitmproxy 1.0, key and value are strings. If the content-type indicates non-form data or the form could not be parsed, this is set to
an empty `MultiDictView`.
Modifications to the MultiDictView update `Request.content`, and vice versa.
""" """
return multidict.MultiDictView( return multidict.MultiDictView(
self._get_urlencoded_form, self._get_urlencoded_form,
@ -895,13 +943,14 @@ class Request(Message):
self.headers["content-type"] = "multipart/form-data" self.headers["content-type"] = "multipart/form-data"
@property @property
def multipart_form(self): def multipart_form(self) -> multidict.MultiDictView[bytes, bytes]:
""" """
The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. The multipart form data.
An empty multidict.MultiDictView if the content-type indicates non-form data
or the content could not be parsed.
Key and value are bytes. If the content-type indicates non-form data or the form could not be parsed, this is set to
an empty `MultiDictView`.
Modifications to the MultiDictView update `Request.content`, and vice versa.
""" """
return multidict.MultiDictView( return multidict.MultiDictView(
self._get_multipart_form, self._get_multipart_form,
@ -977,12 +1026,12 @@ class Response(Message):
headers = headers headers = headers
elif isinstance(headers, dict): elif isinstance(headers, dict):
headers = Headers( headers = Headers(
(always_bytes(k, "utf-8", "surrogateescape"), (always_bytes(k, "utf-8", "surrogateescape"), # type: ignore
always_bytes(v, "utf-8", "surrogateescape")) always_bytes(v, "utf-8", "surrogateescape"))
for k, v in headers.items() for k, v in headers.items()
) )
elif isinstance(headers, Iterable): elif isinstance(headers, Iterable):
headers = Headers(headers) headers = Headers(headers) # type: ignore
else: else:
raise TypeError("Expected headers to be an iterable or dict, but is {}.".format( raise TypeError("Expected headers to be an iterable or dict, but is {}.".format(
type(headers).__name__ type(headers).__name__
@ -1023,7 +1072,8 @@ class Response(Message):
@property @property
def reason(self) -> str: def reason(self) -> str:
""" """
HTTP Reason Phrase, e.g. "Not Found". HTTP reason phrase, for example "Not Found".
HTTP/2 responses do not contain a reason phrase, an empty string will be returned instead. HTTP/2 responses do not contain a reason phrase, an empty string will be returned instead.
""" """
# Encoding: http://stackoverflow.com/a/16674906/934719 # Encoding: http://stackoverflow.com/a/16674906/934719
@ -1049,16 +1099,15 @@ class Response(Message):
self.headers.set_all("set-cookie", cookie_headers) self.headers.set_all("set-cookie", cookie_headers)
@property @property
def cookies(self) -> multidict.MultiDictView: def cookies(self) -> multidict.MultiDictView[str, Tuple[str, multidict.MultiDict[str, Optional[str]]]]:
""" """
The response cookies. A possibly empty The response cookies. A possibly empty `MultiDictView`, where the keys are cookie
:py:class:`~mitmproxy.net.multidict.MultiDictView`, where the keys are cookie name strings, and values are `(cookie value, attributes)` tuples. Within
name strings, and values are (value, attr) tuples. Value is a string, attributes, unary attributes (e.g. `HTTPOnly`) are indicated by a `None` value.
and attr is an MultiDictView containing cookie attributes. Within Modifications to the MultiDictView update `Response.headers`, and vice versa.
attrs, unary attributes (e.g. HTTPOnly) are indicated by a Null value.
Caveats: *Warning:* Changes to `attributes` will not be picked up unless you also reassign
Updating the attr the `(cookie value, attributes)` tuple directly in the `MultiDictView`.
""" """
return multidict.MultiDictView( return multidict.MultiDictView(
self._get_cookies, self._get_cookies,
@ -1074,8 +1123,8 @@ class Response(Message):
This fairly complex and heuristic function refreshes a server This fairly complex and heuristic function refreshes a server
response for replay. response for replay.
- It adjusts date, expires and last-modified headers. - It adjusts date, expires, and last-modified headers.
- It adjusts cookie expiration. - It adjusts cookie expiration.
""" """
if not now: if not now:
now = time.time() now = time.time()
@ -1108,19 +1157,17 @@ class HTTPFlow(flow.Flow):
transaction. transaction.
""" """
request: Request request: Request
"""The client's HTTP request."""
response: Optional[Response] = None response: Optional[Response] = None
"""The server's HTTP response."""
error: Optional[flow.Error] = None error: Optional[flow.Error] = None
""" """
A connection or protocol error affecting this flow.
Note that it's possible for a Flow to have both a response and an error Note that it's possible for a Flow to have both a response and an error
object. This might happen, for instance, when a response was received object. This might happen, for instance, when a response was received
from the server, but there was an error sending it back to the client. from the server, but there was an error sending it back to the client.
""" """
server_conn: connection.Server
client_conn: connection.Client
intercepted: bool = False
""" Is this flow currently being intercepted? """
mode: str
""" What mode was the proxy layer in when receiving this request? """
def __init__(self, client_conn, server_conn, live=None, mode="regular"): def __init__(self, client_conn, server_conn, live=None, mode="regular"):
super().__init__("http", client_conn, server_conn, live) super().__init__("http", client_conn, server_conn, live)
@ -1144,6 +1191,7 @@ class HTTPFlow(flow.Flow):
@property @property
def timestamp_start(self) -> float: def timestamp_start(self) -> float:
"""*Read-only:* An alias for `Request.timestamp_start`."""
return self.request.timestamp_start return self.request.timestamp_start
def copy(self): def copy(self):

21
mitmproxy/net/server_spec.py
View File

@ -1,5 +1,5 @@
""" """
Parse scheme, host and port from a string. Server specs are used to describe an upstream proxy or server.
""" """
import functools import functools
import re import re
@ -31,12 +31,12 @@ def parse(server_spec: str) -> ServerSpec:
""" """
Parses a server mode specification, e.g.: Parses a server mode specification, e.g.:
- http://example.com/ - http://example.com/
- example.org - example.org
- example.com:443 - example.com:443
Raises: *Raises:*
ValueError, if the server specification is invalid. - ValueError, if the server specification is invalid.
""" """
m = server_spec_re.match(server_spec) m = server_spec_re.match(server_spec)
if not m: if not m:
@ -71,13 +71,10 @@ def parse(server_spec: str) -> ServerSpec:
def parse_with_mode(mode: str) -> Tuple[str, ServerSpec]: def parse_with_mode(mode: str) -> Tuple[str, ServerSpec]:
""" """
Parse a proxy mode specification, which is usually just (reverse|upstream):server-spec Parse a proxy mode specification, which is usually just `(reverse|upstream):server-spec`.
Returns: *Raises:*
A (mode, server_spec) tuple. - ValueError, if the specification is invalid.
Raises:
ValueError, if the specification is invalid.
""" """
mode, server_spec = mode.split(":", maxsplit=1) mode, server_spec = mode.split(":", maxsplit=1)
return mode, parse(server_spec) return mode, parse(server_spec)

4
mitmproxy/proxy/server_hooks.py
View File

@ -26,8 +26,12 @@ class ClientDisconnectedHook(commands.StartHook):
@dataclass @dataclass
class ServerConnectionHookData: class ServerConnectionHookData:
"""Event data for server connection event hooks."""
server: connection.Server server: connection.Server
"""The server connection this hook is about."""
client: connection.Client client: connection.Client
"""The client on the other end."""
@dataclass @dataclass

14
mitmproxy/tcp.py
View File

@ -1,5 +1,4 @@
import time import time
from typing import List from typing import List
from mitmproxy import flow from mitmproxy import flow
@ -7,6 +6,12 @@ from mitmproxy.coretypes import serializable
class TCPMessage(serializable.Serializable): class TCPMessage(serializable.Serializable):
"""
An individual TCP "message".
Note that TCP is *stream-based* and not *message-based*.
For practical purposes the stream is chunked into messages here,
but you should not rely on message boundaries.
"""
def __init__(self, from_client, content, timestamp=None): def __init__(self, from_client, content, timestamp=None):
self.from_client = from_client self.from_client = from_client
@ -31,7 +36,6 @@ class TCPMessage(serializable.Serializable):
class TCPFlow(flow.Flow): class TCPFlow(flow.Flow):
""" """
A TCPFlow is a simplified representation of a TCP session. A TCPFlow is a simplified representation of a TCP session.
""" """
@ -45,3 +49,9 @@ class TCPFlow(flow.Flow):
def __repr__(self): def __repr__(self):
return "<TCPFlow ({} messages)>".format(len(self.messages)) return "<TCPFlow ({} messages)>".format(len(self.messages))
__all__ = [
"TCPFlow",
"TCPMessage",
]

67
mitmproxy/websocket.py
View File

@ -1,35 +1,54 @@
import time """
*Deprecation Notice:* Mitmproxy's WebSocket API is going to change soon,
see <https://github.com/mitmproxy/mitmproxy/issues/4425>.
"""
import queue import queue
import time
import warnings import warnings
from typing import List, Optional, Union from typing import List
from typing import Optional
from typing import Union
from mitmproxy import flow
from mitmproxy.coretypes import serializable
from mitmproxy.net import websocket
from mitmproxy.utils import human
from mitmproxy.utils import strutils
from wsproto.frame_protocol import CloseReason from wsproto.frame_protocol import CloseReason
from wsproto.frame_protocol import Opcode from wsproto.frame_protocol import Opcode
from mitmproxy import flow
from mitmproxy.net import websocket
from mitmproxy.coretypes import serializable
from mitmproxy.utils import strutils, human
class WebSocketMessage(serializable.Serializable): class WebSocketMessage(serializable.Serializable):
""" """
A WebSocket message sent from one endpoint to the other. A WebSocket message sent from one endpoint to the other.
""" """
type: Opcode
"""indicates either TEXT or BINARY (from wsproto.frame_protocol.Opcode)."""
from_client: bool
"""True if this messages was sent by the client."""
content: Union[bytes, str]
"""A byte-string representing the content of this message."""
timestamp: float
"""Timestamp of when this message was received or created."""
killed: bool
"""True if this messages was killed and should not be sent to the other endpoint."""
def __init__( def __init__(
self, type: int, from_client: bool, content: Union[bytes, str], timestamp: Optional[float]=None, killed: bool=False self,
type: int,
from_client: bool,
content: Union[bytes, str],
timestamp: Optional[float] = None,
killed: bool = False
) -> None: ) -> None:
self.type = Opcode(type) # type: ignore self.type = Opcode(type) # type: ignore
"""indicates either TEXT or BINARY (from wsproto.frame_protocol.Opcode)."""
self.from_client = from_client self.from_client = from_client
"""True if this messages was sent by the client."""
self.content = content self.content = content
"""A byte-string representing the content of this message."""
self.timestamp: float = timestamp or time.time() self.timestamp: float = timestamp or time.time()
"""Timestamp of when this message was received or created."""
self.killed = killed self.killed = killed
"""True if this messages was killed and should not be sent to the other endpoint."""
@classmethod @classmethod
def from_state(cls, state): def from_state(cls, state):
@ -147,25 +166,3 @@ class WebSocketFlow(flow.Flow):
direction="->" if message.from_client else "<-", direction="->" if message.from_client else "<-",
endpoint=self.handshake_flow.request.path, endpoint=self.handshake_flow.request.path,
) )
def inject_message(self, endpoint, payload):
"""
Inject and send a full WebSocket message to the remote endpoint.
This might corrupt your WebSocket connection! Be careful!
The endpoint needs to be either flow.client_conn or flow.server_conn.
If ``payload`` is of type ``bytes`` then the message is flagged as
being binary If it is of type ``str`` encoded as UTF-8 and sent as
text.
:param payload: The message body to send.
:type payload: ``bytes`` or ``str``
"""
if endpoint == self.client_conn:
self._inject_messages_client.put(payload)
elif endpoint == self.server_conn:
self._inject_messages_server.put(payload)
else:
raise ValueError('Invalid endpoint')

4
setup.cfg
View File

@ -36,6 +36,10 @@ ignore_errors = True
[mypy-test.*] [mypy-test.*]
ignore_errors = True ignore_errors = True
# https://github.com/python/mypy/issues/3004
[mypy-http-modify-form,http-trailers]
ignore_errors = True
[tool:full_coverage] [tool:full_coverage]
exclude = exclude =
mitmproxy/tools/ mitmproxy/tools/

1
setup.py
View File

@ -99,6 +99,7 @@ setup(
'dev': [ 'dev': [
"hypothesis>=5.8,<6.2", "hypothesis>=5.8,<6.2",
"parver>=0.1,<2.0", "parver>=0.1,<2.0",
"pdoc>=4.0.0",
"pytest-asyncio>=0.10.0,<0.14,!=0.14", "pytest-asyncio>=0.10.0,<0.14,!=0.14",
"pytest-cov>=2.7.1,<3", "pytest-cov>=2.7.1,<3",
"pytest-timeout>=1.3.3,<2", "pytest-timeout>=1.3.3,<2",

2
test/examples/test_examples.py
View File

@ -10,7 +10,7 @@ from ..mitmproxy import tservers
class TestScripts(tservers.MasterTest): class TestScripts(tservers.MasterTest):
def test_add_header(self, tdata): def test_add_header(self, tdata):
with taddons.context() as tctx: with taddons.context() as tctx:
a = tctx.script(tdata.path("../examples/addons/scripting-minimal-example.py")) a = tctx.script(tdata.path("../examples/addons/anatomy2.py"))
f = tflow.tflow() f = tflow.tflow()
a.request(f) a.request(f)
assert f.request.headers["myheader"] == "value" assert f.request.headers["myheader"] == "value"

22
test/mitmproxy/test_http.py
View File

@ -751,7 +751,7 @@ class TestHeaders:
headers = Headers() headers = Headers()
assert len(headers) == 0 assert len(headers) == 0
headers = Headers([[b"Host", b"example.com"]]) headers = Headers([(b"Host", b"example.com")])
assert len(headers) == 1 assert len(headers) == 1
assert headers["Host"] == "example.com" assert headers["Host"] == "example.com"
@ -760,14 +760,14 @@ class TestHeaders:
assert headers["Host"] == "example.com" assert headers["Host"] == "example.com"
headers = Headers( headers = Headers(
[[b"Host", b"invalid"]], [(b"Host", b"invalid")],
Host="example.com" Host="example.com"
) )
assert len(headers) == 1 assert len(headers) == 1
assert headers["Host"] == "example.com" assert headers["Host"] == "example.com"
headers = Headers( headers = Headers(
[[b"Host", b"invalid"], [b"Accept", b"text/plain"]], [(b"Host", b"invalid"), (b"Accept", b"text/plain")],
Host="example.com" Host="example.com"
) )
assert len(headers) == 2 assert len(headers) == 2
@ -775,7 +775,7 @@ class TestHeaders:
assert headers["Accept"] == "text/plain" assert headers["Accept"] == "text/plain"
with pytest.raises(TypeError): with pytest.raises(TypeError):
Headers([[b"Host", "not-bytes"]]) Headers([(b"Host", "not-bytes")])
def test_set(self): def test_set(self):
headers = Headers() headers = Headers()
@ -791,8 +791,8 @@ class TestHeaders:
assert bytes(headers) == b"Host: example.com\r\n" assert bytes(headers) == b"Host: example.com\r\n"
headers = Headers([ headers = Headers([
[b"Host", b"example.com"], (b"Host", b"example.com"),
[b"Accept", b"text/plain"] (b"Accept", b"text/plain")
]) ])
assert bytes(headers) == b"Host: example.com\r\nAccept: text/plain\r\n" assert bytes(headers) == b"Host: example.com\r\nAccept: text/plain\r\n"
@ -801,8 +801,8 @@ class TestHeaders:
def test_iter(self): def test_iter(self):
headers = Headers([ headers = Headers([
[b"Set-Cookie", b"foo"], (b"Set-Cookie", b"foo"),
[b"Set-Cookie", b"bar"] (b"Set-Cookie", b"bar")
]) ])
assert list(headers) == ["Set-Cookie"] assert list(headers) == ["Set-Cookie"]
@ -816,9 +816,9 @@ class TestHeaders:
def test_items(self): def test_items(self):
headers = Headers([ headers = Headers([
[b"Set-Cookie", b"foo"], (b"Set-Cookie", b"foo"),
[b"Set-Cookie", b"bar"], (b"Set-Cookie", b"bar"),
[b'Accept', b'text/plain'], (b'Accept', b'text/plain'),
]) ])
assert list(headers.items()) == [ assert list(headers.items()) == [
('Set-Cookie', 'foo, bar'), ('Set-Cookie', 'foo, bar'),

12
test/mitmproxy/test_websocket.py
View File

@ -86,15 +86,3 @@ class TestWebSocketFlow:
d = tflow.twebsocketflow().handshake_flow.get_state() d = tflow.twebsocketflow().handshake_flow.get_state()
tnetstring.dump(d, b) tnetstring.dump(d, b)
assert b.getvalue() assert b.getvalue()
def test_inject_message(self):
f = tflow.twebsocketflow()
with pytest.raises(ValueError):
f.inject_message(None, 'foobar')
f.inject_message(f.client_conn, 'foobar')
assert f._inject_messages_client.qsize() == 1
f.inject_message(f.server_conn, 'foobar')
assert f._inject_messages_client.qsize() == 1