From 805aed4f6a77f8c2950a2b9c9781ec29d3f4f0f6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 9 Feb 2021 01:05:02 +0100 Subject: [PATCH] docs++ --- docs/build.py | 0 docs/ci.sh | 2 +- docs/scripts/api-render.py | 22 ++++---- docs/scripts/examples.py | 29 ++++++++--- docs/scripts/pdoc-template/frame.html.jinja2 | 1 + docs/src/content/addons-api.md | 11 ++-- docs/src/content/addons-examples.md | 2 - docs/src/content/addons-overview.md | 50 ++++++++---------- docs/src/content/addons-scripting.md | 51 ------------------- .../src/content/api/mitmproxy.addonmanager.md | 2 +- docs/src/content/api/mitmproxy.certs.md | 2 +- docs/src/content/api/mitmproxy.connection.md | 2 +- .../api/mitmproxy.coretypes.multidict.md | 2 +- docs/src/content/api/mitmproxy.flow.md | 2 +- docs/src/content/api/mitmproxy.http.md | 2 +- .../content/api/mitmproxy.net.server_spec.md | 11 ++++ .../api/mitmproxy.proxy.server_hooks.md | 2 +- docs/src/content/api/mitmproxy.tcp.md | 2 +- docs/src/content/api/mitmproxy.websocket.md | 2 +- ...ripting-minimal-example.py => anatomy2.py} | 3 ++ examples/addons/http-trailers.py | 1 - examples/addons/internet-in-mirror.py | 14 +++++ examples/addons/internet_in_mirror.py | 12 ----- mitmproxy/addonmanager.py | 5 ++ mitmproxy/addons/dumper.py | 8 +-- mitmproxy/certs.py | 1 + mitmproxy/coretypes/multidict.py | 21 ++++---- mitmproxy/flow.py | 5 +- mitmproxy/http.py | 27 +++++----- mitmproxy/net/server_spec.py | 21 ++++---- mitmproxy/proxy/context.py | 8 --- mitmproxy/proxy/server_hooks.py | 2 + mitmproxy/tcp.py | 2 +- mitmproxy/websocket.py | 6 ++- setup.cfg | 4 ++ test/examples/test_examples.py | 2 +- 36 files changed, 156 insertions(+), 183 deletions(-) mode change 100644 => 100755 docs/build.py delete mode 100644 docs/src/content/addons-scripting.md create mode 100644 docs/src/content/api/mitmproxy.net.server_spec.md rename examples/addons/{scripting-minimal-example.py => anatomy2.py} (53%) create mode 100644 examples/addons/internet-in-mirror.py delete mode 100644 examples/addons/internet_in_mirror.py diff --git a/docs/build.py b/docs/build.py old mode 100644 new mode 100755 diff --git a/docs/ci.sh b/docs/ci.sh index 0d920e090..92597223b 100755 --- a/docs/ci.sh +++ b/docs/ci.sh @@ -6,7 +6,7 @@ set -o pipefail # This script gets run from CI to render and upload docs for the master branch. -./build.sh +./build.py # Only upload if we have defined credentials - we only have these defined for # trusted commits (i.e. not PRs). diff --git a/docs/scripts/api-render.py b/docs/scripts/api-render.py index 1fa95a84b..1991b22dc 100644 --- a/docs/scripts/api-render.py +++ b/docs/scripts/api-render.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os import shutil +import textwrap from pathlib import Path import pdoc.render_helpers @@ -28,6 +29,7 @@ modules = [ "mitmproxy.coretypes.multidict", "mitmproxy.flow", "mitmproxy.http", + "mitmproxy.net.server_spec", "mitmproxy.proxy.server_hooks", "mitmproxy.tcp", "mitmproxy.websocket", @@ -49,17 +51,17 @@ for module in modules: if isinstance(module, Path): continue filename = f"api/{module.replace('.', '/')}.html" - (api_content / f"{module}.md").write_text(f""" ---- -title: "{module}" -url: "{filename}" + (api_content / f"{module}.md").write_text(textwrap.dedent(f""" + --- + title: "{module}" + url: "{filename}" -menu: - addons: - parent: 'API' ---- + menu: + addons: + parent: 'Event Hooks & API' + --- -{{{{< readfile file="/generated/{filename}" >}}}} -""") + {{{{< readfile file="/generated/{filename}" >}}}} + """)) (here / ".." / "src" / "content" / "addons-api.md").touch() diff --git a/docs/scripts/examples.py b/docs/scripts/examples.py index d5abd2ed3..566df2511 100755 --- a/docs/scripts/examples.py +++ b/docs/scripts/examples.py @@ -28,21 +28,38 @@ for example in examples: else: comment = "" overview.append( - f" * [{example.name}](#{slug}){comment}" + f" * [{example.name}](#{slug}){comment}\n" ) listings.append(f"""

Example: {example.name}

```python -{code} +{code.strip()} ``` """) -print("\n".join(overview)) -print(""" -### Community Examples + +print(f""" +# Addon Examples + +### Dedicated Example Addons + +{"".join(overview)} + +### Built-In Addons + +Much of mitmproxy’s own functionality is defined in +[a suite of built-in addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons), +implementing everything from functionality like anticaching and sticky cookies to our onboarding webapp. +The built-in addons make for instructive reading, and you will quickly see that quite complex functionality +can often boil down to a very small, completely self-contained modules. + + +### Additional Community Examples Additional examples contributed by the mitmproxy community can be found [on GitHub](https://github.com/mitmproxy/mitmproxy/tree/master/examples/contrib). +------------------------- + +{"".join(listings)} """) -print("\n".join(listings)) diff --git a/docs/scripts/pdoc-template/frame.html.jinja2 b/docs/scripts/pdoc-template/frame.html.jinja2 index 576dd761b..ce2bf96d5 100644 --- a/docs/scripts/pdoc-template/frame.html.jinja2 +++ b/docs/scripts/pdoc-template/frame.html.jinja2 @@ -1,2 +1,3 @@ {% block style %}{% endblock %} {% block body %}{% endblock %} +
{% block attribution %}{% endblock %}
diff --git a/docs/src/content/addons-api.md b/docs/src/content/addons-api.md index 0903048e1..a0be2b7d2 100644 --- a/docs/src/content/addons-api.md +++ b/docs/src/content/addons-api.md @@ -1,5 +1,5 @@ --- -title: "API" +title: "Event Hooks & API" url: "api/events.html" aliases: - /addons-events/ @@ -9,11 +9,6 @@ menu: weight: 3 --- -# Mitmproxy API - -TODO: Some more text here. - - # Event Hooks Addons hook into mitmproxy's internal mechanisms through event hooks. These are @@ -25,8 +20,8 @@ header with a count of the number of responses seen: {{< example src="examples/addons/http-add-header.py" lang="py" >}} -## Example Addons +## Available Hooks -The following addons showcase all available event hooks. +The following addons list all available event hooks. {{< readfile file="/generated/api/events.html" >}} diff --git a/docs/src/content/addons-examples.md b/docs/src/content/addons-examples.md index 941f189df..9ddf11541 100644 --- a/docs/src/content/addons-examples.md +++ b/docs/src/content/addons-examples.md @@ -6,6 +6,4 @@ menu: weight: 6 --- -# Example Addons - {{< readfile file="/generated/examples.html" markdown="true" >}} diff --git a/docs/src/content/addons-overview.md b/docs/src/content/addons-overview.md index 6aa44a46d..fe7972fdc 100644 --- a/docs/src/content/addons-overview.md +++ b/docs/src/content/addons-overview.md @@ -7,37 +7,19 @@ menu: # Addons -Mitmproxy's addon mechanism consists of a set of APIs that support components of any complexity. Addons interact with -mitmproxy by responding to [events]({{< relref addons-api >}}), which allow them to hook into and change mitmproxy's -behaviour. They are configured through [options]({{< relref addons-options >}}), which can be set in mitmproxy's config -file, changed interactively by users, or passed on the command-line. Finally, they can expose [commands]({{< relref -addons-commands >}}), which allows users to invoke their actions either directly or by binding them to keys in the -interactive tools. +Mitmproxy's addon mechanism is an exceptionally powerful part of mitmproxy. In fact, much of mitmproxy's own +functionality is defined in +[a suite of built-in addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons), +implementing everything from functionality like +[anticaching]({{< relref "overview-features#anticache" >}}) and [sticky cookies]({{< relref +"overview-features#sticky-cookies" >}}) to our onboarding webapp. -Addons are an exceptionally powerful part of mitmproxy. In fact, much of -mitmproxy's own functionality is defined in [a suite of built-in -addons](https://github.com/mitmproxy/mitmproxy/tree/master/mitmproxy/addons), -implementing everything from functionality like [anticaching]({{< relref -"overview-features#anticache" >}}) and [sticky cookies]({{< relref -"overview-features#sticky-cookies" >}}) to our onboarding webapp. The built-in -addons make for instructive reading, and you will quickly see that quite complex -functionality can often boil down to a very small, completely self-contained -modules. Mitmproxy provides the exact same set of facilities it uses for its own -functionality to third-party scripters and extenders. +Addons interact with mitmproxy by responding to [events]({{< relref addons-api >}}), which allow them to hook into and +change mitmproxy's behaviour. They are configured through [options]({{< relref addons-options >}}), which can be set in +mitmproxy's config file, changed interactively by users, or passed on the command-line. Finally, they can expose +[commands]({{< relref addons-commands >}}), which allows users to invoke their actions either directly or by binding +them to keys in the interactive tools. -This document will show you how to build addons using **events**, **options** -and **commands**. However, this is not an API manual, and the mitmproxy source -code remains the canonical reference. One easy way to explore the API from the -command-line is to use [pdoc](https://pdoc.dev/). -Here, for example, is a command that shows the API documentation for the -mitmproxy's HTTP flow classes: - -```bash -pdoc mitmproxy.http -``` - -You will be referring to the mitmproxy API documentation frequently, so keep -**pdoc** or an equivalent handy. # Anatomy of an addon @@ -69,3 +51,13 @@ Here are a few things to note about the code above: first parameter to every event, but we've found it neater to just expose it as an importable global. In this case, we're using the `ctx.log` object to do our logging. + + +# Abbreviated Scripting Syntax + +Sometimes, we would like to write a quick script without going through the trouble of creating a class. +The addons mechanism has a shorthand that allows a module as a whole to be treated as an addon object. +This lets us place event handler functions in the module scope. +For instance, here is a complete script that adds a header to every request: + +{{< example src="examples/addons/anatomy2.py" lang="py" >}} diff --git a/docs/src/content/addons-scripting.md b/docs/src/content/addons-scripting.md deleted file mode 100644 index 5a7ec70f8..000000000 --- a/docs/src/content/addons-scripting.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: "Scripting" -menu: - addons: - weight: 5 ---- - -# Scripting HTTP/1.1 and HTTP/2.0 - -Sometimes, we would like to write a quick script without going through the -trouble of creating a class. The addons mechanism has a shorthand that allows a -module as a whole to be treated as an addon object. This lets us place event -handler functions in the module scope. For instance, here is a complete script -that adds a header to every request. - -{{< example src="examples/addons/scripting-minimal-example.py" lang="py" >}} - - -Here's another example that intercepts requests to a particular URL and sends -an arbitrary response instead: - -{{< example src="examples/addons/http-reply-from-proxy.py" lang="py" >}} - -All events around the HTTP protocol [can be found here]({{< relref "addons-api#http-events">}}). - -For HTTP-related objects, please look at the [http][] module, or the -[Request][], and [Response][] classes for other attributes that you can use when -scripting. - -# Scripting WebSocket - -The WebSocket protocol initially looks like a regular HTTP request, before the client and server agree to upgrade the connection to WebSocket. All scripting events for initial HTTP handshake, and also the dedicated WebSocket events [can be found here]({{< relref "addons-api#websocket-events">}}). - -{{< example src="examples/addons/websocket-simple.py" lang="py" >}} - -For WebSocket-related objects please look at the [websocket][] module to find -all attributes that you can use when scripting. - -[websocket]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/websocket.py - - -# Scripting TCP - -All events around the TCP protocol [can be found here]({{< relref "addons-api#tcp-events">}}). - -{{< example src="examples/addons/tcp-simple.py" lang="py" >}} - -For WebSocket-related objects please look at the [tcp][] module to find -all attributes that you can use when scripting. - -[tcp]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/tcp.py diff --git a/docs/src/content/api/mitmproxy.addonmanager.md b/docs/src/content/api/mitmproxy.addonmanager.md index 091afcb54..fb8bd181b 100644 --- a/docs/src/content/api/mitmproxy.addonmanager.md +++ b/docs/src/content/api/mitmproxy.addonmanager.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/addonmanager.html" menu: addons: - parent: 'API' + parent: 'Event Hooks & API' --- {{< readfile file="/generated/api/mitmproxy/addonmanager.html" >}} diff --git a/docs/src/content/api/mitmproxy.certs.md b/docs/src/content/api/mitmproxy.certs.md index 4ab6aa561..6a3d3d468 100644 --- a/docs/src/content/api/mitmproxy.certs.md +++ b/docs/src/content/api/mitmproxy.certs.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/certs.html" menu: addons: - parent: 'API' + parent: 'Event Hooks & API' --- {{< readfile file="/generated/api/mitmproxy/certs.html" >}} diff --git a/docs/src/content/api/mitmproxy.connection.md b/docs/src/content/api/mitmproxy.connection.md index 82c2d9bb5..5c48cdbd7 100644 --- a/docs/src/content/api/mitmproxy.connection.md +++ b/docs/src/content/api/mitmproxy.connection.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/connection.html" menu: addons: - parent: 'API' + parent: 'Event Hooks & API' --- {{< readfile file="/generated/api/mitmproxy/connection.html" >}} diff --git a/docs/src/content/api/mitmproxy.coretypes.multidict.md b/docs/src/content/api/mitmproxy.coretypes.multidict.md index f0ff3f856..88eb79a51 100644 --- a/docs/src/content/api/mitmproxy.coretypes.multidict.md +++ b/docs/src/content/api/mitmproxy.coretypes.multidict.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/coretypes/multidict.html" menu: addons: - parent: 'API' + parent: 'Event Hooks & API' --- {{< readfile file="/generated/api/mitmproxy/coretypes/multidict.html" >}} diff --git a/docs/src/content/api/mitmproxy.flow.md b/docs/src/content/api/mitmproxy.flow.md index 3124f2717..e07394db1 100644 --- a/docs/src/content/api/mitmproxy.flow.md +++ b/docs/src/content/api/mitmproxy.flow.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/flow.html" menu: addons: - parent: 'API' + parent: 'Event Hooks & API' --- {{< readfile file="/generated/api/mitmproxy/flow.html" >}} diff --git a/docs/src/content/api/mitmproxy.http.md b/docs/src/content/api/mitmproxy.http.md index a9f249e5b..780252096 100644 --- a/docs/src/content/api/mitmproxy.http.md +++ b/docs/src/content/api/mitmproxy.http.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/http.html" menu: addons: - parent: 'API' + parent: 'Event Hooks & API' --- {{< readfile file="/generated/api/mitmproxy/http.html" >}} diff --git a/docs/src/content/api/mitmproxy.net.server_spec.md b/docs/src/content/api/mitmproxy.net.server_spec.md new file mode 100644 index 000000000..5fe2274c9 --- /dev/null +++ b/docs/src/content/api/mitmproxy.net.server_spec.md @@ -0,0 +1,11 @@ + +--- +title: "mitmproxy.net.server_spec" +url: "api/mitmproxy/net/server_spec.html" + +menu: + addons: + parent: 'Event Hooks & API' +--- + +{{< readfile file="/generated/api/mitmproxy/net/server_spec.html" >}} diff --git a/docs/src/content/api/mitmproxy.proxy.server_hooks.md b/docs/src/content/api/mitmproxy.proxy.server_hooks.md index a18806728..bc8ac8e50 100644 --- a/docs/src/content/api/mitmproxy.proxy.server_hooks.md +++ b/docs/src/content/api/mitmproxy.proxy.server_hooks.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/proxy/server_hooks.html" menu: addons: - parent: 'API' + parent: 'Event Hooks & API' --- {{< readfile file="/generated/api/mitmproxy/proxy/server_hooks.html" >}} diff --git a/docs/src/content/api/mitmproxy.tcp.md b/docs/src/content/api/mitmproxy.tcp.md index 2e950350c..fea35b00e 100644 --- a/docs/src/content/api/mitmproxy.tcp.md +++ b/docs/src/content/api/mitmproxy.tcp.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/tcp.html" menu: addons: - parent: 'API' + parent: 'Event Hooks & API' --- {{< readfile file="/generated/api/mitmproxy/tcp.html" >}} diff --git a/docs/src/content/api/mitmproxy.websocket.md b/docs/src/content/api/mitmproxy.websocket.md index 9eda77442..03661320f 100644 --- a/docs/src/content/api/mitmproxy.websocket.md +++ b/docs/src/content/api/mitmproxy.websocket.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/websocket.html" menu: addons: - parent: 'API' + parent: 'Event Hooks & API' --- {{< readfile file="/generated/api/mitmproxy/websocket.html" >}} diff --git a/examples/addons/scripting-minimal-example.py b/examples/addons/anatomy2.py similarity index 53% rename from examples/addons/scripting-minimal-example.py rename to examples/addons/anatomy2.py index 9ecb02931..09814c486 100644 --- a/examples/addons/scripting-minimal-example.py +++ b/examples/addons/anatomy2.py @@ -1,2 +1,5 @@ +"""An addon using the abbreviated scripting syntax.""" + + def request(flow): flow.request.headers["myheader"] = "value" diff --git a/examples/addons/http-trailers.py b/examples/addons/http-trailers.py index 72470de0c..35dc30a86 100644 --- a/examples/addons/http-trailers.py +++ b/examples/addons/http-trailers.py @@ -36,7 +36,6 @@ def request(flow: http.HTTPFlow): def response(flow: http.HTTPFlow): - assert flow.response # make type checker happy if flow.response.trailers: print("HTTP Trailers detected! Response contains:", flow.response.trailers) diff --git a/examples/addons/internet-in-mirror.py b/examples/addons/internet-in-mirror.py new file mode 100644 index 000000000..0c274903e --- /dev/null +++ b/examples/addons/internet-in-mirror.py @@ -0,0 +1,14 @@ +""" +Mirror all web pages. + +Useful if you are living down under. +""" +from mitmproxy import http + + +def response(flow: http.HTTPFlow) -> None: + if flow.response and flow.response.content: + flow.response.content = flow.response.content.replace( + b"", + b"" + ) diff --git a/examples/addons/internet_in_mirror.py b/examples/addons/internet_in_mirror.py deleted file mode 100644 index 8d33cea90..000000000 --- a/examples/addons/internet_in_mirror.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -Mirror all web pages. - -Useful if you are living down under. -""" -from mitmproxy import http - - -def response(flow: http.HTTPFlow) -> None: - assert flow.response # make type checker happy - reflector = b"" - flow.response.content = flow.response.content.replace(b"", reflector) diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index 801a8e90c..a0def682c 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -98,6 +98,11 @@ class Loader: ) def add_command(self, path: str, func: typing.Callable) -> None: + """Add a command to mitmproxy. + + Unless you are generating commands programatically, + this API should be avoided. Decorate your function with `@mitmproxy.command.command` instead. + """ self.master.commands.add(path, func) diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 95807cba0..c47c32dcc 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -80,11 +80,11 @@ class Dumper: def _echo_headers(self, headers: http.Headers): for k, v in headers.fields: - k = strutils.bytes_to_escaped_str(k) - v = strutils.bytes_to_escaped_str(v) + ks = strutils.bytes_to_escaped_str(k) + vs = strutils.bytes_to_escaped_str(v) out = "{}: {}".format( - click.style(k, fg="blue"), - click.style(v) + click.style(ks, fg="blue"), + click.style(vs) ) self.echo(out, ident=4) diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index dd1adb617..afe1bca3b 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -39,6 +39,7 @@ rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI= class Cert(serializable.Serializable): + """Representation of a (TLS) certificate.""" _cert: x509.Certificate def __init__(self, cert: x509.Certificate): diff --git a/mitmproxy/coretypes/multidict.py b/mitmproxy/coretypes/multidict.py index 2fddec1bf..c1ac9f9a4 100644 --- a/mitmproxy/coretypes/multidict.py +++ b/mitmproxy/coretypes/multidict.py @@ -1,9 +1,9 @@ from abc import ABCMeta from abc import abstractmethod -from typing import AbstractSet from typing import Iterator from typing import List from typing import MutableMapping +from typing import Sequence from typing import Tuple from typing import TypeVar @@ -18,7 +18,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta): A MultiDict is a dictionary-like data structure that supports multiple values per key. """ - fields: Tuple[Tuple, ...] + fields: Tuple[Tuple[KT, VT], ...] """The underlying raw datastructure.""" def __repr__(self): @@ -33,7 +33,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta): @staticmethod @abstractmethod - def _reduce_values(values) -> VT: + def _reduce_values(values: Sequence[VT]) -> VT: """ If a user accesses multidict["foo"], this method reduces all values for "foo" to a single value that is returned. @@ -43,7 +43,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta): @staticmethod @abstractmethod - def _kconv(key) -> KT: + def _kconv(key: KT) -> KT: """ This method converts a key to its canonical representation. For example, HTTP headers are case-insensitive, so this method returns key.lower(). @@ -101,7 +101,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta): """ key_kconv = self._kconv(key) - new_fields = [] + new_fields: List[Tuple[KT, VT]] = [] for field in self.fields: if self._kconv(field[0]) == key_kconv: if values: @@ -129,7 +129,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta): item = (key, value) self.fields = self.fields[:index] + (item,) + self.fields[index:] - def keys(self, multi: bool = False) -> Iterator[KT]: + def keys(self, multi: bool = False): """ Get all keys. @@ -141,7 +141,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta): for k, _ in self.items(multi) ) - def values(self, multi: bool = False) -> Iterator[VT]: + def values(self, multi: bool = False): """ Get all values. @@ -153,7 +153,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta): for _, v in self.items(multi) ) - def items(self, multi: bool = False) -> AbstractSet[Tuple[KT,VT]]: + def items(self, multi: bool = False): """ Get all (key, value) tuples. @@ -168,6 +168,7 @@ class _MultiDict(MutableMapping[KT, VT], metaclass=ABCMeta): class MultiDict(_MultiDict[KT, VT], serializable.Serializable): """A concrete MultiDict, storing its own data.""" + def __init__(self, fields=()): super().__init__() self.fields = tuple( @@ -193,7 +194,7 @@ class MultiDict(_MultiDict[KT, VT], serializable.Serializable): return cls(state) -class MultiDictView(_MultiDict): +class MultiDictView(_MultiDict[KT, VT]): """ The MultiDictView provides the MultiDict interface over calculated data. The view itself contains no state - data is retrieved from the parent on @@ -216,7 +217,7 @@ class MultiDictView(_MultiDict): # multiple elements exist with the same key. return values[0] - @property + @property # type: ignore def fields(self): return self._getter() diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index c6100d2ed..d33f637fa 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -66,14 +66,13 @@ class Flow(stateobject.StateObject): server_conn: connection.Server """ The server mitmproxy connected to. - + Some flows may never cause mitmproxy to initiate a server connection, for example because their response is replayed by mitmproxy itself. To simplify implementation, those flows will still have a `server_conn` attribute with a `timestamp_start` set to `None`. """ - error: typing.Optional[Error] = None """A connection or protocol error affecting this flow.""" @@ -91,7 +90,7 @@ class Flow(stateobject.StateObject): is_replay: typing.Optional[str] """ This attribute indicates if this flow has been replayed in either direction. - + - a value of `request` indicates that the request has been artifically replayed by mitmproxy to the server. - a value of `response` indicates that the response to the client's request has been set by server replay. """ diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 50e66f174..1893a9ef5 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -6,7 +6,6 @@ from dataclasses import fields from email.utils import formatdate from email.utils import mktime_tz from email.utils import parsedate_tz -from typing import AbstractSet from typing import Callable from typing import Dict from typing import Iterable @@ -44,7 +43,8 @@ def _always_bytes(x: Union[str, bytes]) -> bytes: return strutils.always_bytes(x, "utf-8", "surrogateescape") -class Headers(multidict.MultiDict[str, str]): +# This cannot be easily typed with mypy yet, so we just specify MultiDict without concrete types. +class Headers(multidict.MultiDict): # type: ignore """ Header class which allows both convenient access to individual headers as well as direct access to the underlying raw data. Provides a full dictionary interface. @@ -107,11 +107,10 @@ class Headers(multidict.MultiDict[str, str]): raise TypeError("Header fields must be bytes.") # content_type -> content-type - headers = { + self.update({ _always_bytes(name).replace(b"_", b"-"): _always_bytes(value) for name, value in headers.items() - } - self.update(headers) + }) fields: Tuple[Tuple[bytes, bytes], ...] @@ -131,7 +130,7 @@ class Headers(multidict.MultiDict[str, str]): else: return b"" - def __delitem__(self, key: str) -> None: + def __delitem__(self, key: Union[str, bytes]) -> None: key = _always_bytes(key) super().__delitem__(key) @@ -166,7 +165,7 @@ class Headers(multidict.MultiDict[str, str]): value = _always_bytes(value) super().insert(index, key, value) - def items(self, multi=False) -> AbstractSet[Tuple[str, str]]: + def items(self, multi=False): if multi: return ( (_native(k), _native(v)) @@ -249,7 +248,7 @@ class Message(serializable.Serializable): but immediately streamed to the destination instead. Alternatively, a transformation function can be specified, but please note that packet should not be relied upon. - + This attribute must be set in the `requestheaders` or `responseheaders` hook. Setting it in `request` or `response` is already too late, mitmproxy has buffered the message body already. """ @@ -421,7 +420,7 @@ class Message(serializable.Serializable): enc = self._guess_encoding() try: - self.content = encoding.encode(text, enc) + self.content = cast(bytes, encoding.encode(text, enc)) except ValueError: # Fall back to UTF-8 and update the content-type header. ct = parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {}) @@ -581,7 +580,7 @@ class Request(Message): for k, v in headers.items() ) elif isinstance(headers, Iterable): - headers = Headers(headers) + headers = Headers(headers) # type: ignore else: raise TypeError("Expected headers to be an iterable or dict, but is {}.".format( type(headers).__name__ @@ -860,7 +859,7 @@ class Request(Message): return tuple(url.unquote(i) for i in path.split("/") if i) @path_components.setter - def path_components(self, components: Tuple[str, ...]): + def path_components(self, components: Iterable[str]): components = map(lambda x: url.quote(x, safe=""), components) path = "/" + "/".join(components) _, _, _, params, query, fragment = urllib.parse.urlparse(self.url) @@ -1027,12 +1026,12 @@ class Response(Message): headers = headers elif isinstance(headers, dict): headers = Headers( - (always_bytes(k, "utf-8", "surrogateescape"), + (always_bytes(k, "utf-8", "surrogateescape"), # type: ignore always_bytes(v, "utf-8", "surrogateescape")) for k, v in headers.items() ) elif isinstance(headers, Iterable): - headers = Headers(headers) + headers = Headers(headers) # type: ignore else: raise TypeError("Expected headers to be an iterable or dict, but is {}.".format( type(headers).__name__ @@ -1164,7 +1163,7 @@ class HTTPFlow(flow.Flow): error: Optional[flow.Error] = None """ A connection or protocol error affecting this flow. - + Note that it's possible for a Flow to have both a response and an error object. This might happen, for instance, when a response was received from the server, but there was an error sending it back to the client. diff --git a/mitmproxy/net/server_spec.py b/mitmproxy/net/server_spec.py index 75c0942cb..f117774f5 100644 --- a/mitmproxy/net/server_spec.py +++ b/mitmproxy/net/server_spec.py @@ -1,5 +1,5 @@ """ -Parse scheme, host and port from a string. +Server specs are used to describe an upstream proxy or server. """ import functools import re @@ -31,12 +31,12 @@ def parse(server_spec: str) -> ServerSpec: """ Parses a server mode specification, e.g.: - - http://example.com/ - - example.org - - example.com:443 + - http://example.com/ + - example.org + - example.com:443 - Raises: - ValueError, if the server specification is invalid. + *Raises:* + - ValueError, if the server specification is invalid. """ m = server_spec_re.match(server_spec) if not m: @@ -71,13 +71,10 @@ def parse(server_spec: str) -> ServerSpec: def parse_with_mode(mode: str) -> Tuple[str, ServerSpec]: """ - Parse a proxy mode specification, which is usually just (reverse|upstream):server-spec + Parse a proxy mode specification, which is usually just `(reverse|upstream):server-spec`. - Returns: - A (mode, server_spec) tuple. - - Raises: - ValueError, if the specification is invalid. + *Raises:* + - ValueError, if the specification is invalid. """ mode, server_spec = mode.split(":", maxsplit=1) return mode, parse(server_spec) diff --git a/mitmproxy/proxy/context.py b/mitmproxy/proxy/context.py index 396c8b9df..6a21504d9 100644 --- a/mitmproxy/proxy/context.py +++ b/mitmproxy/proxy/context.py @@ -32,11 +32,3 @@ class Context: ret.server = self.server ret.layers = self.layers.copy() return ret - - -__all__ = [ - "Connection", - "Client", - "Server", - "ConnectionState", -] diff --git a/mitmproxy/proxy/server_hooks.py b/mitmproxy/proxy/server_hooks.py index b98708224..7afba799f 100644 --- a/mitmproxy/proxy/server_hooks.py +++ b/mitmproxy/proxy/server_hooks.py @@ -26,6 +26,8 @@ class ClientDisconnectedHook(commands.StartHook): @dataclass class ServerConnectionHookData: + """Event data for server connection event hooks.""" + server: connection.Server """The server connection this hook is about.""" client: connection.Client diff --git a/mitmproxy/tcp.py b/mitmproxy/tcp.py index 11c9e01c3..ec3c78b67 100644 --- a/mitmproxy/tcp.py +++ b/mitmproxy/tcp.py @@ -1,5 +1,4 @@ import time - from typing import List from mitmproxy import flow @@ -51,6 +50,7 @@ class TCPFlow(flow.Flow): def __repr__(self): return "".format(len(self.messages)) + __all__ = [ "TCPFlow", "TCPMessage", diff --git a/mitmproxy/websocket.py b/mitmproxy/websocket.py index d96d0e262..e7137c0d8 100644 --- a/mitmproxy/websocket.py +++ b/mitmproxy/websocket.py @@ -1,3 +1,7 @@ +""" +*Deprecation Notice:* Mitmproxy's WebSocket API is going to change soon, +see . +""" import queue import time import warnings @@ -24,7 +28,7 @@ class WebSocketMessage(serializable.Serializable): """indicates either TEXT or BINARY (from wsproto.frame_protocol.Opcode).""" from_client: bool """True if this messages was sent by the client.""" - content: bytes + content: Union[bytes, str] """A byte-string representing the content of this message.""" timestamp: float """Timestamp of when this message was received or created.""" diff --git a/setup.cfg b/setup.cfg index 5dfd81d4d..0550512e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,6 +36,10 @@ ignore_errors = True [mypy-test.*] ignore_errors = True +# https://github.com/python/mypy/issues/3004 +[mypy-http-modify-form,http-trailers] +ignore_errors = True + [tool:full_coverage] exclude = mitmproxy/tools/ diff --git a/test/examples/test_examples.py b/test/examples/test_examples.py index ec4ca517c..32211fe88 100644 --- a/test/examples/test_examples.py +++ b/test/examples/test_examples.py @@ -10,7 +10,7 @@ from ..mitmproxy import tservers class TestScripts(tservers.MasterTest): def test_add_header(self, tdata): with taddons.context() as tctx: - a = tctx.script(tdata.path("../examples/addons/scripting-minimal-example.py")) + a = tctx.script(tdata.path("../examples/addons/anatomy2.py")) f = tflow.tflow() a.request(f) assert f.request.headers["myheader"] == "value"