This commit is contained in:
Maximilian Hils 2021-02-09 01:05:02 +01:00
parent 0ab59e5524
commit 805aed4f6a
36 changed files with 156 additions and 183 deletions

0
docs/build.py Normal file → Executable file
View File

View File

@ -6,7 +6,7 @@ set -o pipefail
# This script gets run from CI to render and upload docs for the master branch.
./build.sh
./build.py
# Only upload if we have defined credentials - we only have these defined for
# trusted commits (i.e. not PRs).

View File

@ -1,6 +1,7 @@
#!/usr/bin/env python3
import os
import shutil
import textwrap
from pathlib import Path
import pdoc.render_helpers
@ -28,6 +29,7 @@ modules = [
"mitmproxy.coretypes.multidict",
"mitmproxy.flow",
"mitmproxy.http",
"mitmproxy.net.server_spec",
"mitmproxy.proxy.server_hooks",
"mitmproxy.tcp",
"mitmproxy.websocket",
@ -49,17 +51,17 @@ for module in modules:
if isinstance(module, Path):
continue
filename = f"api/{module.replace('.', '/')}.html"
(api_content / f"{module}.md").write_text(f"""
---
title: "{module}"
url: "{filename}"
(api_content / f"{module}.md").write_text(textwrap.dedent(f"""
---
title: "{module}"
url: "{filename}"
menu:
menu:
addons:
parent: 'API'
---
parent: 'Event Hooks & API'
---
{{{{< readfile file="/generated/{filename}" >}}}}
""")
{{{{< readfile file="/generated/{filename}" >}}}}
"""))
(here / ".." / "src" / "content" / "addons-api.md").touch()

View File

@ -28,21 +28,38 @@ for example in examples:
else:
comment = ""
overview.append(
f" * [{example.name}](#{slug}){comment}"
f" * [{example.name}](#{slug}){comment}\n"
)
listings.append(f"""
<h3 id="{slug}">Example: {example.name}</h3>
```python
{code}
{code.strip()}
```
""")
print("\n".join(overview))
print("""
### Community Examples
print(f"""
# Addon Examples
### Dedicated Example Addons
{"".join(overview)}
### Built-In Addons
Much of mitmproxys own functionality is defined in
[a suite of built-in addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons),
implementing everything from functionality like anticaching and sticky cookies to our onboarding webapp.
The built-in addons make for instructive reading, and you will quickly see that quite complex functionality
can often boil down to a very small, completely self-contained modules.
### Additional Community Examples
Additional examples contributed by the mitmproxy community can be found
[on GitHub](https://github.com/mitmproxy/mitmproxy/tree/master/examples/contrib).
-------------------------
{"".join(listings)}
""")
print("\n".join(listings))

View File

@ -1,2 +1,3 @@
{% block style %}{% endblock %}
{% block body %}{% endblock %}
<div class="pdoc" style="margin-top: 4rem">{% block attribution %}{% endblock %}</div>

View File

@ -1,5 +1,5 @@
---
title: "API"
title: "Event Hooks & API"
url: "api/events.html"
aliases:
- /addons-events/
@ -9,11 +9,6 @@ menu:
weight: 3
---
# Mitmproxy API
TODO: Some more text here.
# Event Hooks
Addons hook into mitmproxy's internal mechanisms through event hooks. These are
@ -25,8 +20,8 @@ header with a count of the number of responses seen:
{{< example src="examples/addons/http-add-header.py" lang="py" >}}
## Example Addons
## Available Hooks
The following addons showcase all available event hooks.
The following addons list all available event hooks.
{{< readfile file="/generated/api/events.html" >}}

View File

@ -6,6 +6,4 @@ menu:
weight: 6
---
# Example Addons
{{< readfile file="/generated/examples.html" markdown="true" >}}

View File

@ -7,37 +7,19 @@ menu:
# Addons
Mitmproxy's addon mechanism consists of a set of APIs that support components of any complexity. Addons interact with
mitmproxy by responding to [events]({{< relref addons-api >}}), which allow them to hook into and change mitmproxy's
behaviour. They are configured through [options]({{< relref addons-options >}}), which can be set in mitmproxy's config
file, changed interactively by users, or passed on the command-line. Finally, they can expose [commands]({{< relref
addons-commands >}}), which allows users to invoke their actions either directly or by binding them to keys in the
interactive tools.
Mitmproxy's addon mechanism is an exceptionally powerful part of mitmproxy. In fact, much of mitmproxy's own
functionality is defined in
[a suite of built-in addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons),
implementing everything from functionality like
[anticaching]({{< relref "overview-features#anticache" >}}) and [sticky cookies]({{< relref
"overview-features#sticky-cookies" >}}) to our onboarding webapp.
Addons are an exceptionally powerful part of mitmproxy. In fact, much of
mitmproxy's own functionality is defined in [a suite of built-in
addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons),
implementing everything from functionality like [anticaching]({{< relref
"overview-features#anticache" >}}) and [sticky cookies]({{< relref
"overview-features#sticky-cookies" >}}) to our onboarding webapp. The built-in
addons make for instructive reading, and you will quickly see that quite complex
functionality can often boil down to a very small, completely self-contained
modules. Mitmproxy provides the exact same set of facilities it uses for its own
functionality to third-party scripters and extenders.
Addons interact with mitmproxy by responding to [events]({{< relref addons-api >}}), which allow them to hook into and
change mitmproxy's behaviour. They are configured through [options]({{< relref addons-options >}}), which can be set in
mitmproxy's config file, changed interactively by users, or passed on the command-line. Finally, they can expose
[commands]({{< relref addons-commands >}}), which allows users to invoke their actions either directly or by binding
them to keys in the interactive tools.
This document will show you how to build addons using **events**, **options**
and **commands**. However, this is not an API manual, and the mitmproxy source
code remains the canonical reference. One easy way to explore the API from the
command-line is to use [pdoc](https://pdoc.dev/).
Here, for example, is a command that shows the API documentation for the
mitmproxy's HTTP flow classes:
```bash
pdoc mitmproxy.http
```
You will be referring to the mitmproxy API documentation frequently, so keep
**pdoc** or an equivalent handy.
# Anatomy of an addon
@ -69,3 +51,13 @@ Here are a few things to note about the code above:
first parameter to every event, but we've found it neater to just expose it as
an importable global. In this case, we're using the `ctx.log` object to do our
logging.
# Abbreviated Scripting Syntax
Sometimes, we would like to write a quick script without going through the trouble of creating a class.
The addons mechanism has a shorthand that allows a module as a whole to be treated as an addon object.
This lets us place event handler functions in the module scope.
For instance, here is a complete script that adds a header to every request:
{{< example src="examples/addons/anatomy2.py" lang="py" >}}

View File

@ -1,51 +0,0 @@
---
title: "Scripting"
menu:
addons:
weight: 5
---
# Scripting HTTP/1.1 and HTTP/2.0
Sometimes, we would like to write a quick script without going through the
trouble of creating a class. The addons mechanism has a shorthand that allows a
module as a whole to be treated as an addon object. This lets us place event
handler functions in the module scope. For instance, here is a complete script
that adds a header to every request.
{{< example src="examples/addons/scripting-minimal-example.py" lang="py" >}}
Here's another example that intercepts requests to a particular URL and sends
an arbitrary response instead:
{{< example src="examples/addons/http-reply-from-proxy.py" lang="py" >}}
All events around the HTTP protocol [can be found here]({{< relref "addons-api#http-events">}}).
For HTTP-related objects, please look at the [http][] module, or the
[Request][], and [Response][] classes for other attributes that you can use when
scripting.
# Scripting WebSocket
The WebSocket protocol initially looks like a regular HTTP request, before the client and server agree to upgrade the connection to WebSocket. All scripting events for initial HTTP handshake, and also the dedicated WebSocket events [can be found here]({{< relref "addons-api#websocket-events">}}).
{{< example src="examples/addons/websocket-simple.py" lang="py" >}}
For WebSocket-related objects please look at the [websocket][] module to find
all attributes that you can use when scripting.
[websocket]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/websocket.py
# Scripting TCP
All events around the TCP protocol [can be found here]({{< relref "addons-api#tcp-events">}}).
{{< example src="examples/addons/tcp-simple.py" lang="py" >}}
For WebSocket-related objects please look at the [tcp][] module to find
all attributes that you can use when scripting.
[tcp]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/tcp.py

View File

@ -5,7 +5,7 @@ url: "api/mitmproxy/addonmanager.html"
menu:
addons:
parent: 'API'
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/addonmanager.html" >}}

View File

@ -5,7 +5,7 @@ url: "api/mitmproxy/certs.html"
menu:
addons:
parent: 'API'
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/certs.html" >}}

View File

@ -5,7 +5,7 @@ url: "api/mitmproxy/connection.html"
menu:
addons:
parent: 'API'
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/connection.html" >}}

View File

@ -5,7 +5,7 @@ url: "api/mitmproxy/coretypes/multidict.html"
menu:
addons:
parent: 'API'
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/coretypes/multidict.html" >}}

View File

@ -5,7 +5,7 @@ url: "api/mitmproxy/flow.html"
menu:
addons:
parent: 'API'
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/flow.html" >}}

View File

@ -5,7 +5,7 @@ url: "api/mitmproxy/http.html"
menu:
addons:
parent: 'API'
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/http.html" >}}

View File

@ -0,0 +1,11 @@
---
title: "mitmproxy.net.server_spec"
url: "api/mitmproxy/net/server_spec.html"
menu:
addons:
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/net/server_spec.html" >}}

View File

@ -5,7 +5,7 @@ url: "api/mitmproxy/proxy/server_hooks.html"
menu:
addons:
parent: 'API'
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/proxy/server_hooks.html" >}}

View File

@ -5,7 +5,7 @@ url: "api/mitmproxy/tcp.html"
menu:
addons:
parent: 'API'
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/tcp.html" >}}

View File

@ -5,7 +5,7 @@ url: "api/mitmproxy/websocket.html"
menu:
addons:
parent: 'API'
parent: 'Event Hooks & API'
---
{{< readfile file="/generated/api/mitmproxy/websocket.html" >}}

View File

@ -1,2 +1,5 @@
"""An addon using the abbreviated scripting syntax."""
def request(flow):
flow.request.headers["myheader"] = "value"

View File

@ -36,7 +36,6 @@ def request(flow: http.HTTPFlow):
def response(flow: http.HTTPFlow):
assert flow.response # make type checker happy
if flow.response.trailers:
print("HTTP Trailers detected! Response contains:", flow.response.trailers)

View File

@ -0,0 +1,14 @@
"""
Mirror all web pages.
Useful if you are living down under.
"""
from mitmproxy import http
def response(flow: http.HTTPFlow) -> None:
if flow.response and flow.response.content:
flow.response.content = flow.response.content.replace(
b"</head>",
b"<style>body {transform: scaleX(-1);}</style></head>"
)

View File

@ -1,12 +0,0 @@
"""
Mirror all web pages.
Useful if you are living down under.
"""
from mitmproxy import http
def response(flow: http.HTTPFlow) -> None:
assert flow.response # make type checker happy
reflector = b"<style>body {transform: scaleX(-1);}</style></head>"
flow.response.content = flow.response.content.replace(b"</head>", reflector)

View File

@ -98,6 +98,11 @@ class Loader:
)
def add_command(self, path: str, func: typing.Callable) -> None:
"""Add a command to mitmproxy.
Unless you are generating commands programatically,
this API should be avoided. Decorate your function with `@mitmproxy.command.command` instead.
"""
self.master.commands.add(path, func)

View File

@ -80,11 +80,11 @@ class Dumper:
def _echo_headers(self, headers: http.Headers):
for k, v in headers.fields:
k = strutils.bytes_to_escaped_str(k)
v = strutils.bytes_to_escaped_str(v)
ks = strutils.bytes_to_escaped_str(k)
vs = strutils.bytes_to_escaped_str(v)
out = "{}: {}".format(
click.style(k, fg="blue"),
click.style(v)
click.style(ks, fg="blue"),
click.style(vs)
)
self.echo(out, ident=4)

View File

@ -39,6 +39,7 @@ rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI=
class Cert(serializable.Serializable):
"""Representation of a (TLS) certificate."""
_cert: x509.Certificate
def __init__(self, cert: x509.Certificate):

View File

@ -1,9 +1,9 @@
from abc import ABCMeta
from abc import abstractmethod
from typing import AbstractSet
from typing import Iterator
from typing import List
from typing import MutableMapping
from typing import Sequence
from typing import Tuple
from typing import TypeVar
@ -18,7 +18,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta):
A MultiDict is a dictionary-like data structure that supports multiple values per key.
"""
fields: Tuple[Tuple, ...]
fields: Tuple[Tuple[KT, VT], ...]
"""The underlying raw datastructure."""
def __repr__(self):
@ -33,7 +33,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta):
@staticmethod
@abstractmethod
def _reduce_values(values) -> VT:
def _reduce_values(values: Sequence[VT]) -> VT:
"""
If a user accesses multidict["foo"], this method
reduces all values for "foo" to a single value that is returned.
@ -43,7 +43,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta):
@staticmethod
@abstractmethod
def _kconv(key) -> KT:
def _kconv(key: KT) -> KT:
"""
This method converts a key to its canonical representation.
For example, HTTP headers are case-insensitive, so this method returns key.lower().
@ -101,7 +101,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta):
"""
key_kconv = self._kconv(key)
new_fields = []
new_fields: List[Tuple[KT, VT]] = []
for field in self.fields:
if self._kconv(field[0]) == key_kconv:
if values:
@ -129,7 +129,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta):
item = (key, value)
self.fields = self.fields[:index] + (item,) + self.fields[index:]
def keys(self, multi: bool = False) -> Iterator[KT]:
def keys(self, multi: bool = False):
"""
Get all keys.
@ -141,7 +141,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta):
for k, _ in self.items(multi)
)
def values(self, multi: bool = False) -> Iterator[VT]:
def values(self, multi: bool = False):
"""
Get all values.
@ -153,7 +153,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta):
for _, v in self.items(multi)
)
def items(self, multi: bool = False) -> AbstractSet[Tuple[KT,VT]]:
def items(self, multi: bool = False):
"""
Get all (key, value) tuples.
@ -168,6 +168,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta):
class MultiDict(_MultiDict[KT, VT], serializable.Serializable):
"""A concrete MultiDict, storing its own data."""
def __init__(self, fields=()):
super().__init__()
self.fields = tuple(
@ -193,7 +194,7 @@ class MultiDict(_MultiDict[KT, VT], serializable.Serializable):
return cls(state)
class MultiDictView(_MultiDict):
class MultiDictView(_MultiDict[KT, VT]):
"""
The MultiDictView provides the MultiDict interface over calculated data.
The view itself contains no state - data is retrieved from the parent on
@ -216,7 +217,7 @@ class MultiDictView(_MultiDict):
# multiple elements exist with the same key.
return values[0]
@property
@property # type: ignore
def fields(self):
return self._getter()

View File

@ -73,7 +73,6 @@ class Flow(stateobject.StateObject):
with a `timestamp_start` set to `None`.
"""
error: typing.Optional[Error] = None
"""A connection or protocol error affecting this flow."""

View File

@ -6,7 +6,6 @@ from dataclasses import fields
from email.utils import formatdate
from email.utils import mktime_tz
from email.utils import parsedate_tz
from typing import AbstractSet
from typing import Callable
from typing import Dict
from typing import Iterable
@ -44,7 +43,8 @@ def _always_bytes(x: Union[str, bytes]) -> bytes:
return strutils.always_bytes(x, "utf-8", "surrogateescape")
class Headers(multidict.MultiDict[str, str]):
# This cannot be easily typed with mypy yet, so we just specify MultiDict without concrete types.
class Headers(multidict.MultiDict): # type: ignore
"""
Header class which allows both convenient access to individual headers as well as
direct access to the underlying raw data. Provides a full dictionary interface.
@ -107,11 +107,10 @@ class Headers(multidict.MultiDict[str, str]):
raise TypeError("Header fields must be bytes.")
# content_type -> content-type
headers = {
self.update({
_always_bytes(name).replace(b"_", b"-"): _always_bytes(value)
for name, value in headers.items()
}
self.update(headers)
})
fields: Tuple[Tuple[bytes, bytes], ...]
@ -131,7 +130,7 @@ class Headers(multidict.MultiDict[str, str]):
else:
return b""
def __delitem__(self, key: str) -> None:
def __delitem__(self, key: Union[str, bytes]) -> None:
key = _always_bytes(key)
super().__delitem__(key)
@ -166,7 +165,7 @@ class Headers(multidict.MultiDict[str, str]):
value = _always_bytes(value)
super().insert(index, key, value)
def items(self, multi=False) -> AbstractSet[Tuple[str, str]]:
def items(self, multi=False):
if multi:
return (
(_native(k), _native(v))
@ -421,7 +420,7 @@ class Message(serializable.Serializable):
enc = self._guess_encoding()
try:
self.content = encoding.encode(text, enc)
self.content = cast(bytes, encoding.encode(text, enc))
except ValueError:
# Fall back to UTF-8 and update the content-type header.
ct = parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {})
@ -581,7 +580,7 @@ class Request(Message):
for k, v in headers.items()
)
elif isinstance(headers, Iterable):
headers = Headers(headers)
headers = Headers(headers) # type: ignore
else:
raise TypeError("Expected headers to be an iterable or dict, but is {}.".format(
type(headers).__name__
@ -860,7 +859,7 @@ class Request(Message):
return tuple(url.unquote(i) for i in path.split("/") if i)
@path_components.setter
def path_components(self, components: Tuple[str, ...]):
def path_components(self, components: Iterable[str]):
components = map(lambda x: url.quote(x, safe=""), components)
path = "/" + "/".join(components)
_, _, _, params, query, fragment = urllib.parse.urlparse(self.url)
@ -1027,12 +1026,12 @@ class Response(Message):
headers = headers
elif isinstance(headers, dict):
headers = Headers(
(always_bytes(k, "utf-8", "surrogateescape"),
(always_bytes(k, "utf-8", "surrogateescape"), # type: ignore
always_bytes(v, "utf-8", "surrogateescape"))
for k, v in headers.items()
)
elif isinstance(headers, Iterable):
headers = Headers(headers)
headers = Headers(headers) # type: ignore
else:
raise TypeError("Expected headers to be an iterable or dict, but is {}.".format(
type(headers).__name__

View File

@ -1,5 +1,5 @@
"""
Parse scheme, host and port from a string.
Server specs are used to describe an upstream proxy or server.
"""
import functools
import re
@ -35,8 +35,8 @@ def parse(server_spec: str) -> ServerSpec:
- example.org
- example.com:443
Raises:
ValueError, if the server specification is invalid.
*Raises:*
- ValueError, if the server specification is invalid.
"""
m = server_spec_re.match(server_spec)
if not m:
@ -71,13 +71,10 @@ def parse(server_spec: str) -> ServerSpec:
def parse_with_mode(mode: str) -> Tuple[str, ServerSpec]:
"""
Parse a proxy mode specification, which is usually just (reverse|upstream):server-spec
Parse a proxy mode specification, which is usually just `(reverse|upstream):server-spec`.
Returns:
A (mode, server_spec) tuple.
Raises:
ValueError, if the specification is invalid.
*Raises:*
- ValueError, if the specification is invalid.
"""
mode, server_spec = mode.split(":", maxsplit=1)
return mode, parse(server_spec)

View File

@ -32,11 +32,3 @@ class Context:
ret.server = self.server
ret.layers = self.layers.copy()
return ret
__all__ = [
"Connection",
"Client",
"Server",
"ConnectionState",
]

View File

@ -26,6 +26,8 @@ class ClientDisconnectedHook(commands.StartHook):
@dataclass
class ServerConnectionHookData:
"""Event data for server connection event hooks."""
server: connection.Server
"""The server connection this hook is about."""
client: connection.Client

View File

@ -1,5 +1,4 @@
import time
from typing import List
from mitmproxy import flow
@ -51,6 +50,7 @@ class TCPFlow(flow.Flow):
def __repr__(self):
return "<TCPFlow ({} messages)>".format(len(self.messages))
__all__ = [
"TCPFlow",
"TCPMessage",

View File

@ -1,3 +1,7 @@
"""
*Deprecation Notice:* Mitmproxy's WebSocket API is going to change soon,
see <https://github.com/mitmproxy/mitmproxy/issues/4425>.
"""
import queue
import time
import warnings
@ -24,7 +28,7 @@ class WebSocketMessage(serializable.Serializable):
"""indicates either TEXT or BINARY (from wsproto.frame_protocol.Opcode)."""
from_client: bool
"""True if this messages was sent by the client."""
content: bytes
content: Union[bytes, str]
"""A byte-string representing the content of this message."""
timestamp: float
"""Timestamp of when this message was received or created."""

View File

@ -36,6 +36,10 @@ ignore_errors = True
[mypy-test.*]
ignore_errors = True
# https://github.com/python/mypy/issues/3004
[mypy-http-modify-form,http-trailers]
ignore_errors = True
[tool:full_coverage]
exclude =
mitmproxy/tools/

View File

@ -10,7 +10,7 @@ from ..mitmproxy import tservers
class TestScripts(tservers.MasterTest):
def test_add_header(self, tdata):
with taddons.context() as tctx:
a = tctx.script(tdata.path("../examples/addons/scripting-minimal-example.py"))
a = tctx.script(tdata.path("../examples/addons/anatomy2.py"))
f = tflow.tflow()
a.request(f)
assert f.request.headers["myheader"] == "value"