From a7d1f32c89f3ea8b989a2d63fac13a44bc34f946 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 4 Feb 2021 20:22:36 +0100 Subject: [PATCH] docs: first prototype to link from events to API reference --- docs/build.py | 17 +++ docs/build.sh | 20 --- docs/modd.conf | 4 +- docs/scripts/api-events.py | 142 ++++++++++++++++++ docs/scripts/{api.py => api-render.py} | 17 +-- docs/scripts/events.py | 142 ------------------ docs/scripts/pdoc-template/module.html.jinja2 | 33 ++++ docs/src/assets/style.scss | 3 + .../{addons-events.md => addons-api.md} | 19 ++- docs/src/content/addons-scripting.md | 6 +- docs/src/content/api/_index.md | 10 -- docs/src/content/api/mitmproxy.flow.md | 2 +- docs/src/content/api/mitmproxy.http.md | 2 +- .../content/api/mitmproxy.proxy.context.md | 2 +- docs/src/content/api/mitmproxy.tcp.md | 2 +- docs/src/content/api/mitmproxy.websocket.md | 2 +- 16 files changed, 222 insertions(+), 201 deletions(-) create mode 100644 docs/build.py delete mode 100755 docs/build.sh create mode 100644 docs/scripts/api-events.py rename docs/scripts/{api.py => api-render.py} (84%) mode change 100755 => 100644 delete mode 100755 docs/scripts/events.py rename docs/src/content/{addons-events.md => addons-api.md} (68%) delete mode 100644 docs/src/content/api/_index.md diff --git a/docs/build.py b/docs/build.py new file mode 100644 index 000000000..479841e80 --- /dev/null +++ b/docs/build.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +import shutil +import subprocess +from pathlib import Path + + +here = Path(__file__).parent + +for script in (here / "scripts").glob("*.py"): + print(f"Generating output for {script.name}...") + out = subprocess.check_output(["python3", script.absolute()], text=True) + if out: + (here / "src" / "generated" / f"{script.stem}.html").write_text(out) + +if (here / "public").exists(): + shutil.rmtree(here / "public") +subprocess.run(["hugo"], cwd=here / "src") diff --git a/docs/build.sh b/docs/build.sh deleted file mode 100755 index eaeb87907..000000000 --- a/docs/build.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o pipefail -set -o nounset -# set -o xtrace - -SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" -pushd ${SCRIPTPATH} - -for script in scripts/*.py ; do - output="${script##*/}" - output="src/generated/${output%.*}.html" - echo "Generating output for ${script} into ${output} ..." - "${script}" > "${output}" -done - -rm -rf ./public -cd src -hugo diff --git a/docs/modd.conf b/docs/modd.conf index 3f16cdb59..f3fd350bc 100644 --- a/docs/modd.conf +++ b/docs/modd.conf @@ -1,5 +1,5 @@ -scripts/*.py { - prep: ./build.sh +scripts/** { + prep: python3 build.py } { diff --git a/docs/scripts/api-events.py b/docs/scripts/api-events.py new file mode 100644 index 000000000..31b0f3301 --- /dev/null +++ b/docs/scripts/api-events.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +import contextlib +import inspect +import textwrap +from pathlib import Path +from typing import List, Type + +import mitmproxy.addons.next_layer # noqa +from mitmproxy import hooks, log, addonmanager +from mitmproxy.proxy import server_hooks, layer +from mitmproxy.proxy.layers import http, tcp, tls, websocket + +known = set() + + +def category(name: str, hooks: List[Type[hooks.Hook]]) -> None: + all_params = [ + list(inspect.signature(hook.__init__).parameters.values())[1:] + for hook in hooks + ] + + # slightly overengineered, but this was fun to write. ¯\_(ツ)_/¯ + imports = set() + types = set() + for params in all_params: + for param in params: + try: + mod = inspect.getmodule(param.annotation).__name__ + if mod == "typing": + # this is ugly, but can be removed once we are on Python 3.9+ only + imports.add(inspect.getmodule(param.annotation.__args__[0]).__name__) + types.add(param.annotation._name) + else: + imports.add(mod) + except AttributeError: + raise ValueError(f"Missing type annotation: {params}") + imports.discard("builtins") + if types: + print(f"from typing import {', '.join(sorted(types))}") + print("from mitmproxy import ctx") + for imp in sorted(imports): + print(f"import {imp}") + print() + + print(f"class {name}_Events:") + + first = True + for hook, params in zip(hooks, all_params): + if first: + first = False + else: + print() + if hook.name in known: + raise RuntimeError(f"Already documented: {hook}") + known.add(hook.name) + doc = inspect.getdoc(hook) + print(f" def {hook.name}({', '.join(str(p) for p in params)}):") + print(textwrap.indent(f'"""\n{doc}\n"""', " ")) + if params: + print(f' ctx.log(f"{hook.name}: {" ".join("{" + p.name + "=}" for p in params)}")') + else: + print(f' ctx.log("{hook.name}")') + print("") + + +outfile = Path(__file__).parent.parent / "src" / "generated" / "events.py" + +with outfile.open("w") as f, contextlib.redirect_stdout(f): + + category( + "Lifecycle", + [ + addonmanager.LoadHook, + hooks.RunningHook, + hooks.ConfigureHook, + hooks.DoneHook, + ] + ) + + category( + "Connection", + [ + server_hooks.ClientConnectedHook, + server_hooks.ClientDisconnectedHook, + server_hooks.ServerConnectHook, + server_hooks.ServerConnectedHook, + server_hooks.ServerDisconnectedHook, + ] + ) + + category( + "HTTP", + [ + http.HttpRequestHeadersHook, + http.HttpRequestHook, + http.HttpResponseHeadersHook, + http.HttpResponseHook, + http.HttpErrorHook, + http.HttpConnectHook, + ] + ) + + category( + "TCP", + [ + tcp.TcpStartHook, + tcp.TcpMessageHook, + tcp.TcpEndHook, + tcp.TcpErrorHook, + ] + ) + + category( + "TLS", + [ + tls.TlsClienthelloHook, + tls.TlsStartHook, + ] + ) + + category( + "WebSocket", + [ + websocket.WebsocketStartHook, + websocket.WebsocketMessageHook, + websocket.WebsocketEndHook, + websocket.WebsocketErrorHook, + ] + ) + + category( + "Advanced_Lifecycle", + [ + layer.NextLayerHook, + hooks.UpdateHook, + log.AddLogHook, + ] + ) + +not_documented = set(hooks.all_hooks.keys()) - known +if not_documented: + raise RuntimeError(f"Not documented: {not_documented}") diff --git a/docs/scripts/api.py b/docs/scripts/api-render.py old mode 100755 new mode 100644 similarity index 84% rename from docs/scripts/api.py rename to docs/scripts/api-render.py index e2af27450..11f5714cc --- a/docs/scripts/api.py +++ b/docs/scripts/api-render.py @@ -20,6 +20,7 @@ pdoc.render.configure( ) modules = [ + here / ".." / "src" / "generated" / "events.py", "mitmproxy.proxy.context", "mitmproxy.http", "mitmproxy.flow", @@ -39,6 +40,8 @@ if api_content.exists(): api_content.mkdir() for module in modules: + if isinstance(module, Path): + continue filename = f"api/{ module.replace('.','/') }.html" (api_content / f"{module}.md").write_text(f""" --- @@ -47,20 +50,8 @@ url: "{filename}" menu: addons: - parent: 'API Reference' + parent: 'API' --- {{{{< readfile file="/generated/{filename}" >}}}} """) - -(api_content / f"_index.md").write_text(f""" ---- -title: "API Reference" -layout: single -menu: - addons: - weight: 5 ---- - -# API Reference -""") diff --git a/docs/scripts/events.py b/docs/scripts/events.py deleted file mode 100755 index 6462c6341..000000000 --- a/docs/scripts/events.py +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env python3 -import inspect -import textwrap -from typing import List, Type - -import mitmproxy.addons.next_layer # noqa -from mitmproxy import hooks, log, addonmanager -from mitmproxy.proxy import server_hooks, layer -from mitmproxy.proxy.layers import http, tcp, tls, websocket - -known = set() - - -def category(name: str, hooks: List[Type[hooks.Hook]]) -> None: - print(f"### {name} Events") - print("```python") - - all_params = [ - list(inspect.signature(hook.__init__).parameters.values())[1:] - for hook in hooks - ] - - # slightly overengineered, but this was fun to write. ¯\_(ツ)_/¯ - imports = set() - types = set() - for params in all_params: - for param in params: - try: - mod = inspect.getmodule(param.annotation).__name__ - if mod == "typing": - # this is ugly, but can be removed once we are on Python 3.9+ only - imports.add(inspect.getmodule(param.annotation.__args__[0]).__name__) - types.add(param.annotation._name) - else: - imports.add(mod) - except AttributeError: - raise ValueError(f"Missing type annotation: {params}") - imports.discard("builtins") - if types: - print(f"from typing import {', '.join(sorted(types))}") - print("from mitmproxy import ctx") - for imp in sorted(imports): - print(f"import {imp}") - print() - - first = True - for hook, params in zip(hooks, all_params): - if first: - first = False - else: - print() - if hook.name in known: - raise RuntimeError(f"Already documented: {hook}") - known.add(hook.name) - doc = inspect.getdoc(hook) - print(f"def {hook.name}({', '.join(str(p) for p in params)}):") - print(textwrap.indent(f'"""\n{doc}\n"""', " ")) - if params: - print(f' ctx.log(f"{hook.name}: {" ".join("{" + p.name + "=}" for p in params)}")') - else: - print(f' ctx.log("{hook.name}")') - print("```") - - -category( - "Lifecycle", - [ - addonmanager.LoadHook, - hooks.RunningHook, - hooks.ConfigureHook, - hooks.DoneHook, - ] -) - -category( - "Connection", - [ - server_hooks.ClientConnectedHook, - server_hooks.ClientDisconnectedHook, - server_hooks.ServerConnectHook, - server_hooks.ServerConnectedHook, - server_hooks.ServerDisconnectedHook, - ] -) - -category( - "HTTP", - [ - http.HttpRequestHeadersHook, - http.HttpRequestHook, - http.HttpResponseHeadersHook, - http.HttpResponseHook, - http.HttpErrorHook, - http.HttpConnectHook, - ] -) - -category( - "TCP", - [ - tcp.TcpStartHook, - tcp.TcpMessageHook, - tcp.TcpEndHook, - tcp.TcpErrorHook, - ] -) - -category( - "TLS", - [ - tls.TlsClienthelloHook, - tls.TlsStartHook, - ] -) - -category( - "WebSocket", - [ - websocket.WebsocketStartHook, - websocket.WebsocketMessageHook, - websocket.WebsocketEndHook, - websocket.WebsocketErrorHook, - ] -) - -category( - "Advanced Lifecycle", - [ - layer.NextLayerHook, - hooks.UpdateHook, - log.AddLogHook, - ] -) - -not_documented = set(hooks.all_hooks.keys()) - known -if not_documented: - raise RuntimeError(f"Not documented: {not_documented}") - -# print("") -# for i in flowfilter.help: -# print("" % i) -# print("
%s%s
") diff --git a/docs/scripts/pdoc-template/module.html.jinja2 b/docs/scripts/pdoc-template/module.html.jinja2 index 728a9b4dc..4aebdc4f4 100644 --- a/docs/scripts/pdoc-template/module.html.jinja2 +++ b/docs/scripts/pdoc-template/module.html.jinja2 @@ -1,3 +1,36 @@ {% extends "default/module.html.jinja2" %} {% block nav %}{% endblock %} {% block style_layout %}{% endblock %} + +{# +We do a bit of hackery here: generated/events.py is automatically created by scripts/api-events.py, +and then documented using a heavily customized style. +#} +{% if module.name == "events" %} + {% macro module_name() %} + + {% endmacro %} + {% macro view_source(doc) %} + {% if doc.type == "function" %} +
+ View Source + {{ doc.source | dedent | highlight }} +
+ {% endif %} + {% endmacro %} + {% macro is_public(doc) %} + {% if doc.name != "__init__" %} + {{ default_is_public(doc) }} + {% endif %} + {% endmacro %} + {% macro class(cls) %} +

+ {{ headerlink(cls) }} + {{ cls.name.replace("_", " ") }} +

+ {% endmacro %} +{% endif %} diff --git a/docs/src/assets/style.scss b/docs/src/assets/style.scss index e5254b541..818680358 100644 --- a/docs/src/assets/style.scss +++ b/docs/src/assets/style.scss @@ -7,6 +7,9 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox $panel-heading-size: 1em; $panel-heading-weight: 600; +$menu-list-link-padding: .3em .75em; +$menu-label-spacing: .7em; +$menu-nested-list-margin: .3em .75em; /*!* bulma.io v0.8.0 | MIT License | github.com/jgthms/bulma */ diff --git a/docs/src/content/addons-events.md b/docs/src/content/addons-api.md similarity index 68% rename from docs/src/content/addons-events.md rename to docs/src/content/addons-api.md index eb58ddb6c..eaa18f45f 100644 --- a/docs/src/content/addons-events.md +++ b/docs/src/content/addons-api.md @@ -1,10 +1,19 @@ --- -title: "Event Hooks" +title: "API" +url: "api/events.html" +aliases: + - /addons-events/ +layout: single menu: addons: - weight: 2 + weight: 3 --- +# Mitmproxy API + +TODO: Some more text here. + + # Event Hooks Addons hook into mitmproxy's internal mechanisms through event hooks. These are @@ -16,9 +25,7 @@ header with a count of the number of responses seen: {{< example src="examples/addons/http-add-header.py" lang="py" >}} -## Supported Events +## Addon Events -Below we list events supported by mitmproxy. We've added -annotations to illustrate the argument types. -{{< readfile file="/generated/events.html" markdown="true" >}} +{{< readfile file="/generated/api/events.html" >}} diff --git a/docs/src/content/addons-scripting.md b/docs/src/content/addons-scripting.md index 343f635c1..5a7ec70f8 100644 --- a/docs/src/content/addons-scripting.md +++ b/docs/src/content/addons-scripting.md @@ -21,7 +21,7 @@ an arbitrary response instead: {{< example src="examples/addons/http-reply-from-proxy.py" lang="py" >}} -All events around the HTTP protocol [can be found here]({{< relref "addons-events#http-events">}}). +All events around the HTTP protocol [can be found here]({{< relref "addons-api#http-events">}}). For HTTP-related objects, please look at the [http][] module, or the [Request][], and [Response][] classes for other attributes that you can use when @@ -29,7 +29,7 @@ scripting. # Scripting WebSocket -The WebSocket protocol initially looks like a regular HTTP request, before the client and server agree to upgrade the connection to WebSocket. All scripting events for initial HTTP handshake, and also the dedicated WebSocket events [can be found here]({{< relref "addons-events#websocket-events">}}). +The WebSocket protocol initially looks like a regular HTTP request, before the client and server agree to upgrade the connection to WebSocket. All scripting events for initial HTTP handshake, and also the dedicated WebSocket events [can be found here]({{< relref "addons-api#websocket-events">}}). {{< example src="examples/addons/websocket-simple.py" lang="py" >}} @@ -41,7 +41,7 @@ all attributes that you can use when scripting. # Scripting TCP -All events around the TCP protocol [can be found here]({{< relref "addons-events#tcp-events">}}). +All events around the TCP protocol [can be found here]({{< relref "addons-api#tcp-events">}}). {{< example src="examples/addons/tcp-simple.py" lang="py" >}} diff --git a/docs/src/content/api/_index.md b/docs/src/content/api/_index.md deleted file mode 100644 index 911e96639..000000000 --- a/docs/src/content/api/_index.md +++ /dev/null @@ -1,10 +0,0 @@ - ---- -title: "API Reference" -layout: single -menu: - addons: - weight: 5 ---- - -# API Reference diff --git a/docs/src/content/api/mitmproxy.flow.md b/docs/src/content/api/mitmproxy.flow.md index 0837998c1..3124f2717 100644 --- a/docs/src/content/api/mitmproxy.flow.md +++ b/docs/src/content/api/mitmproxy.flow.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/flow.html" menu: addons: - parent: 'API Reference' + parent: 'API' --- {{< readfile file="/generated/api/mitmproxy/flow.html" >}} diff --git a/docs/src/content/api/mitmproxy.http.md b/docs/src/content/api/mitmproxy.http.md index 69496d5c2..a9f249e5b 100644 --- a/docs/src/content/api/mitmproxy.http.md +++ b/docs/src/content/api/mitmproxy.http.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/http.html" menu: addons: - parent: 'API Reference' + parent: 'API' --- {{< readfile file="/generated/api/mitmproxy/http.html" >}} diff --git a/docs/src/content/api/mitmproxy.proxy.context.md b/docs/src/content/api/mitmproxy.proxy.context.md index e11c6c494..a11abe674 100644 --- a/docs/src/content/api/mitmproxy.proxy.context.md +++ b/docs/src/content/api/mitmproxy.proxy.context.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/proxy/context.html" menu: addons: - parent: 'API Reference' + parent: 'API' --- {{< readfile file="/generated/api/mitmproxy/proxy/context.html" >}} diff --git a/docs/src/content/api/mitmproxy.tcp.md b/docs/src/content/api/mitmproxy.tcp.md index 59553d49d..2e950350c 100644 --- a/docs/src/content/api/mitmproxy.tcp.md +++ b/docs/src/content/api/mitmproxy.tcp.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/tcp.html" menu: addons: - parent: 'API Reference' + parent: 'API' --- {{< readfile file="/generated/api/mitmproxy/tcp.html" >}} diff --git a/docs/src/content/api/mitmproxy.websocket.md b/docs/src/content/api/mitmproxy.websocket.md index 0f70619d5..9eda77442 100644 --- a/docs/src/content/api/mitmproxy.websocket.md +++ b/docs/src/content/api/mitmproxy.websocket.md @@ -5,7 +5,7 @@ url: "api/mitmproxy/websocket.html" menu: addons: - parent: 'API Reference' + parent: 'API' --- {{< readfile file="/generated/api/mitmproxy/websocket.html" >}}