docs: first prototype to link from events to API reference

docs/build.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+import shutil
+import subprocess
+from pathlib import Path
+
+
+here = Path(__file__).parent
+
+for script in (here / "scripts").glob("*.py"):
+    print(f"Generating output for {script.name}...")
+    out = subprocess.check_output(["python3", script.absolute()], text=True)
+    if out:
+        (here / "src" / "generated" / f"{script.stem}.html").write_text(out)
+
+if (here / "public").exists():
+    shutil.rmtree(here / "public")
+subprocess.run(["hugo"], cwd=here / "src")

docs/build.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o pipefail
-set -o nounset
-# set -o xtrace
-
-SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
-pushd ${SCRIPTPATH}
-
-for script in scripts/*.py ; do
-  output="${script##*/}"
-  output="src/generated/${output%.*}.html"
-  echo "Generating output for ${script} into ${output} ..."
-  "${script}" > "${output}"
-done
-
-rm -rf ./public
-cd src
-hugo

docs/modd.conf

@@ -1,5 +1,5 @@
-scripts/*.py {
+scripts/** {
-    prep: ./build.sh
+    prep: python3 build.py
 }
 
 {

docs/scripts/api-events.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+import contextlib
+import inspect
+import textwrap
+from pathlib import Path
+from typing import List, Type
+
+import mitmproxy.addons.next_layer  # noqa
+from mitmproxy import hooks, log, addonmanager
+from mitmproxy.proxy import server_hooks, layer
+from mitmproxy.proxy.layers import http, tcp, tls, websocket
+
+known = set()
+
+
+def category(name: str, hooks: List[Type[hooks.Hook]]) -> None:
+    all_params = [
+        list(inspect.signature(hook.__init__).parameters.values())[1:]
+        for hook in hooks
+    ]
+
+    # slightly overengineered, but this was fun to write.  ¯\_(ツ)_/¯
+    imports = set()
+    types = set()
+    for params in all_params:
+        for param in params:
+            try:
+                mod = inspect.getmodule(param.annotation).__name__
+                if mod == "typing":
+                    # this is ugly, but can be removed once we are on Python 3.9+ only
+                    imports.add(inspect.getmodule(param.annotation.__args__[0]).__name__)
+                    types.add(param.annotation._name)
+                else:
+                    imports.add(mod)
+            except AttributeError:
+                raise ValueError(f"Missing type annotation: {params}")
+    imports.discard("builtins")
+    if types:
+        print(f"from typing import {', '.join(sorted(types))}")
+    print("from mitmproxy import ctx")
+    for imp in sorted(imports):
+        print(f"import {imp}")
+    print()
+
+    print(f"class {name}_Events:")
+
+    first = True
+    for hook, params in zip(hooks, all_params):
+        if first:
+            first = False
+        else:
+            print()
+        if hook.name in known:
+            raise RuntimeError(f"Already documented: {hook}")
+        known.add(hook.name)
+        doc = inspect.getdoc(hook)
+        print(f"    def {hook.name}({', '.join(str(p) for p in params)}):")
+        print(textwrap.indent(f'"""\n{doc}\n"""', "        "))
+        if params:
+            print(f'        ctx.log(f"{hook.name}: {" ".join("{" + p.name + "=}" for p in params)}")')
+        else:
+            print(f'        ctx.log("{hook.name}")')
+    print("")
+
+
+outfile = Path(__file__).parent.parent / "src" / "generated" / "events.py"
+
+with outfile.open("w") as f, contextlib.redirect_stdout(f):
+
+    category(
+        "Lifecycle",
+        [
+            addonmanager.LoadHook,
+            hooks.RunningHook,
+            hooks.ConfigureHook,
+            hooks.DoneHook,
+        ]
+    )
+
+    category(
+        "Connection",
+        [
+            server_hooks.ClientConnectedHook,
+            server_hooks.ClientDisconnectedHook,
+            server_hooks.ServerConnectHook,
+            server_hooks.ServerConnectedHook,
+            server_hooks.ServerDisconnectedHook,
+        ]
+    )
+
+    category(
+        "HTTP",
+        [
+            http.HttpRequestHeadersHook,
+            http.HttpRequestHook,
+            http.HttpResponseHeadersHook,
+            http.HttpResponseHook,
+            http.HttpErrorHook,
+            http.HttpConnectHook,
+        ]
+    )
+
+    category(
+        "TCP",
+        [
+            tcp.TcpStartHook,
+            tcp.TcpMessageHook,
+            tcp.TcpEndHook,
+            tcp.TcpErrorHook,
+        ]
+    )
+
+    category(
+        "TLS",
+        [
+            tls.TlsClienthelloHook,
+            tls.TlsStartHook,
+        ]
+    )
+
+    category(
+        "WebSocket",
+        [
+            websocket.WebsocketStartHook,
+            websocket.WebsocketMessageHook,
+            websocket.WebsocketEndHook,
+            websocket.WebsocketErrorHook,
+        ]
+    )
+
+    category(
+        "Advanced_Lifecycle",
+        [
+            layer.NextLayerHook,
+            hooks.UpdateHook,
+            log.AddLogHook,
+        ]
+    )
+
+not_documented = set(hooks.all_hooks.keys()) - known
+if not_documented:
+    raise RuntimeError(f"Not documented: {not_documented}")

docs/scripts/api-render.py

@@ -20,6 +20,7 @@ pdoc.render.configure(
 )
 
 modules = [
+    here / ".." / "src" / "generated" / "events.py",
     "mitmproxy.proxy.context",
     "mitmproxy.http",
     "mitmproxy.flow",
@@ -39,6 +40,8 @@ if api_content.exists():
 api_content.mkdir()
 
 for module in modules:
+    if isinstance(module, Path):
+        continue
     filename = f"api/{ module.replace('.','/') }.html"
     (api_content / f"{module}.md").write_text(f"""
 ---
@@ -47,20 +50,8 @@ url: "{filename}"
 
 menu:
     addons:
-        parent: 'API Reference'
+        parent: 'API'
 ---
 
 {{{{< readfile file="/generated/{filename}" >}}}}
 """)
-
-(api_content / f"_index.md").write_text(f"""
----
-title: "API Reference"
-layout: single
-menu:
-    addons:
-        weight: 5
----
-
-# API Reference
-""")

docs/scripts/events.py

@@ -1,142 +0,0 @@
-#!/usr/bin/env python3
-import inspect
-import textwrap
-from typing import List, Type
-
-import mitmproxy.addons.next_layer  # noqa
-from mitmproxy import hooks, log, addonmanager
-from mitmproxy.proxy import server_hooks, layer
-from mitmproxy.proxy.layers import http, tcp, tls, websocket
-
-known = set()
-
-
-def category(name: str, hooks: List[Type[hooks.Hook]]) -> None:
-    print(f"### {name} Events")
-    print("```python")
-
-    all_params = [
-        list(inspect.signature(hook.__init__).parameters.values())[1:]
-        for hook in hooks
-    ]
-
-    # slightly overengineered, but this was fun to write.  ¯\_(ツ)_/¯
-    imports = set()
-    types = set()
-    for params in all_params:
-        for param in params:
-            try:
-                mod = inspect.getmodule(param.annotation).__name__
-                if mod == "typing":
-                    # this is ugly, but can be removed once we are on Python 3.9+ only
-                    imports.add(inspect.getmodule(param.annotation.__args__[0]).__name__)
-                    types.add(param.annotation._name)
-                else:
-                    imports.add(mod)
-            except AttributeError:
-                raise ValueError(f"Missing type annotation: {params}")
-    imports.discard("builtins")
-    if types:
-        print(f"from typing import {', '.join(sorted(types))}")
-    print("from mitmproxy import ctx")
-    for imp in sorted(imports):
-        print(f"import {imp}")
-    print()
-
-    first = True
-    for hook, params in zip(hooks, all_params):
-        if first:
-            first = False
-        else:
-            print()
-        if hook.name in known:
-            raise RuntimeError(f"Already documented: {hook}")
-        known.add(hook.name)
-        doc = inspect.getdoc(hook)
-        print(f"def {hook.name}({', '.join(str(p) for p in params)}):")
-        print(textwrap.indent(f'"""\n{doc}\n"""', "    "))
-        if params:
-            print(f'    ctx.log(f"{hook.name}: {" ".join("{" + p.name + "=}" for p in params)}")')
-        else:
-            print(f'    ctx.log("{hook.name}")')
-    print("```")
-
-
-category(
-    "Lifecycle",
-    [
-        addonmanager.LoadHook,
-        hooks.RunningHook,
-        hooks.ConfigureHook,
-        hooks.DoneHook,
-    ]
-)
-
-category(
-    "Connection",
-    [
-        server_hooks.ClientConnectedHook,
-        server_hooks.ClientDisconnectedHook,
-        server_hooks.ServerConnectHook,
-        server_hooks.ServerConnectedHook,
-        server_hooks.ServerDisconnectedHook,
-    ]
-)
-
-category(
-    "HTTP",
-    [
-        http.HttpRequestHeadersHook,
-        http.HttpRequestHook,
-        http.HttpResponseHeadersHook,
-        http.HttpResponseHook,
-        http.HttpErrorHook,
-        http.HttpConnectHook,
-    ]
-)
-
-category(
-    "TCP",
-    [
-        tcp.TcpStartHook,
-        tcp.TcpMessageHook,
-        tcp.TcpEndHook,
-        tcp.TcpErrorHook,
-    ]
-)
-
-category(
-    "TLS",
-    [
-        tls.TlsClienthelloHook,
-        tls.TlsStartHook,
-    ]
-)
-
-category(
-    "WebSocket",
-    [
-        websocket.WebsocketStartHook,
-        websocket.WebsocketMessageHook,
-        websocket.WebsocketEndHook,
-        websocket.WebsocketErrorHook,
-    ]
-)
-
-category(
-    "Advanced Lifecycle",
-    [
-        layer.NextLayerHook,
-        hooks.UpdateHook,
-        log.AddLogHook,
-    ]
-)
-
-not_documented = set(hooks.all_hooks.keys()) - known
-if not_documented:
-    raise RuntimeError(f"Not documented: {not_documented}")
-
-# print("<table class=\"table filtertable\"><tbody>")
-# for i in flowfilter.help:
-#    print("<tr><th>%s</th><td>%s</td></tr>" % i)
-# print("</tbody></table>")

docs/scripts/pdoc-template/module.html.jinja2

@@ -1,3 +1,36 @@
 {% extends "default/module.html.jinja2" %}
 {% block nav %}{% endblock %}
 {% block style_layout %}{% endblock %}
+
+{#
+We do a bit of hackery here: generated/events.py is automatically created by scripts/api-events.py,
+and then documented using a heavily customized style.
+#}
+{% if module.name == "events" %}
+    {% macro module_name() %}
+        <style type="text/css">
+            .pdoc .classattr {
+                margin-left: 0 !important;
+            }
+        </style>
+    {% endmacro %}
+    {% macro view_source(doc) %}
+        {% if doc.type == "function" %}
+            <details>
+                <summary>View Source</summary>
+                {{ doc.source | dedent | highlight }}
+            </details>
+        {% endif %}
+    {% endmacro %}
+    {% macro is_public(doc) %}
+        {% if doc.name != "__init__" %}
+            {{ default_is_public(doc) }}
+        {% endif %}
+    {% endmacro %}
+    {% macro class(cls) %}
+        <h3>
+            {{ headerlink(cls) }}
+            <strong>{{ cls.name.replace("_", " ") }}</strong>
+        </h3>
+    {% endmacro %}
+{% endif %}

docs/src/assets/style.scss

@@ -7,6 +7,9 @@ $family-sans-serif: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Ox
 
 $panel-heading-size: 1em;
 $panel-heading-weight: 600;
+$menu-list-link-padding: .3em .75em;
+$menu-label-spacing: .7em;
+$menu-nested-list-margin: .3em .75em;
 
 /*!*
 bulma.io v0.8.0 | MIT License | github.com/jgthms/bulma */

docs/src/content/addons-api.md

@@ -1,10 +1,19 @@
 ---
-title: "Event Hooks"
+title: "API"
+url: "api/events.html"
+aliases:
+    - /addons-events/
+layout: single
 menu:
     addons:
-        weight: 2
+        weight: 3
 ---
 
+# Mitmproxy API
+
+TODO: Some more text here.
+
+
 # Event Hooks
 
 Addons hook into mitmproxy's internal mechanisms through event hooks. These are
@@ -16,9 +25,7 @@ header with a count of the number of responses seen:
 {{< example src="examples/addons/http-add-header.py" lang="py" >}}
 
 
-## Supported Events
+## Addon Events
 
-Below we list events supported by mitmproxy. We've added
-annotations to illustrate the argument types.
 
-{{< readfile file="/generated/events.html" markdown="true" >}}
+{{< readfile file="/generated/api/events.html" >}}

docs/src/content/addons-scripting.md

@@ -21,7 +21,7 @@ an arbitrary response instead:
 
 {{< example src="examples/addons/http-reply-from-proxy.py" lang="py" >}}
 
-All events around the HTTP protocol [can be found here]({{< relref "addons-events#http-events">}}).
+All events around the HTTP protocol [can be found here]({{< relref "addons-api#http-events">}}).
 
 For HTTP-related objects, please look at the [http][] module, or the
 [Request][], and [Response][] classes for other attributes that you can use when
@@ -29,7 +29,7 @@ scripting.
 
 # Scripting WebSocket
 
-The WebSocket protocol initially looks like a regular HTTP request, before the client and server agree to upgrade the connection to WebSocket. All scripting events for initial HTTP handshake, and also the dedicated WebSocket events [can be found here]({{< relref "addons-events#websocket-events">}}).
+The WebSocket protocol initially looks like a regular HTTP request, before the client and server agree to upgrade the connection to WebSocket. All scripting events for initial HTTP handshake, and also the dedicated WebSocket events [can be found here]({{< relref "addons-api#websocket-events">}}).
 
 {{< example src="examples/addons/websocket-simple.py" lang="py" >}}
 
@@ -41,7 +41,7 @@ all attributes that you can use when scripting.
 
 # Scripting TCP
 
-All events around the TCP protocol [can be found here]({{< relref "addons-events#tcp-events">}}).
+All events around the TCP protocol [can be found here]({{< relref "addons-api#tcp-events">}}).
 
 {{< example src="examples/addons/tcp-simple.py" lang="py" >}}
 

docs/src/content/api/_index.md

@@ -1,10 +0,0 @@
-
----
-title: "API Reference"
-layout: single
-menu:
-    addons:
-        weight: 5
----
-
-# API Reference

docs/src/content/api/mitmproxy.flow.md

@@ -5,7 +5,7 @@ url: "api/mitmproxy/flow.html"
 
 menu:
     addons:
-        parent: 'API Reference'
+        parent: 'API'
 ---
 
 {{< readfile file="/generated/api/mitmproxy/flow.html" >}}

docs/src/content/api/mitmproxy.http.md

@@ -5,7 +5,7 @@ url: "api/mitmproxy/http.html"
 
 menu:
     addons:
-        parent: 'API Reference'
+        parent: 'API'
 ---
 
 {{< readfile file="/generated/api/mitmproxy/http.html" >}}

docs/src/content/api/mitmproxy.proxy.context.md

@@ -5,7 +5,7 @@ url: "api/mitmproxy/proxy/context.html"
 
 menu:
     addons:
-        parent: 'API Reference'
+        parent: 'API'
 ---
 
 {{< readfile file="/generated/api/mitmproxy/proxy/context.html" >}}

docs/src/content/api/mitmproxy.tcp.md

@@ -5,7 +5,7 @@ url: "api/mitmproxy/tcp.html"
 
 menu:
     addons:
-        parent: 'API Reference'
+        parent: 'API'
 ---
 
 {{< readfile file="/generated/api/mitmproxy/tcp.html" >}}

docs/src/content/api/mitmproxy.websocket.md

@@ -5,7 +5,7 @@ url: "api/mitmproxy/websocket.html"
 
 menu:
     addons:
-        parent: 'API Reference'
+        parent: 'API'
 ---
 
 {{< readfile file="/generated/api/mitmproxy/websocket.html" >}}