add cli recording scripts

docs/build.sh

@@ -8,7 +8,7 @@ set -o nounset
 SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
 pushd ${SCRIPTPATH}
 
-for script in scripts/* ; do
+for script in scripts/*.py ; do
   output="${script##*/}"
   output="src/generated/${output%.*}.html"
   echo "Generating output for ${script} into ${output} ..."

docs/scripts/clirecording/Dockerfile

@@ -0,0 +1,45 @@
+# todo: use a more lightweight base, e.g., Alpine Linux
+FROM ubuntu:18.04
+
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+ENV TERM screen-256color
+
+# install mitmproxy, asciinema, and dependencies
+RUN apt-get update && apt-get install -y \
+    asciinema \
+    autoconf \
+    automake \
+    autotools-dev \
+    bison \
+    curl \
+    git \
+    libevent-dev \
+    libtool \
+    locales \
+    m4 \
+    make \
+    ncurses-dev \
+    pkg-config \
+    python3-pip \
+    python3 \
+    wget \
+    xterm \
+    && locale-gen --purge "en_US.UTF-8" \
+    && update-locale "LANG=en_US.UTF-8" \
+    && pip3 install libtmux curl requests mitmproxy
+
+# install latest tmux (to support popups)
+RUN git clone https://github.com/tmux/tmux.git \
+    && cd tmux \
+    && sh autogen.sh \
+    && ./configure && make && make install
+
+WORKDIR /root/clidirector
+
+COPY ./docker/tmux.conf ../.tmux.conf
+COPY clidirector.py screenplays.py record.py ./
+
+RUN echo 'PS1="[tutorial@mitmproxy] $ "' >> /root/.bashrc
+
+ENTRYPOINT [ "./record.py" ]

docs/scripts/clirecording/clidirector.py

@@ -0,0 +1,170 @@
+import datetime
+import json
+import libtmux
+import random
+import requests
+import subprocess
+import threading
+import time
+import typing
+
+
+class CliDirector:
+    def __init__(self):
+        self.record_start = None
+        self.pause_between_keys = 0.1
+        self.pause_between_keys_rand = 0.25
+
+    def start(self, filename: str, width: int = 0, height: int = 0) -> libtmux.Session:
+        self.start_session(width, height)
+        self.start_recording(filename)
+        return self.tmux_session
+
+    def start_session(self, width: int = 0, height: int = 0) -> libtmux.Session:
+        self.tmux_server = libtmux.Server()
+        self.tmux_session = self.tmux_server.new_session(session_name="asciinema_recorder", kill_session=True)
+        self.tmux_pane = self.tmux_session.attached_window.attached_pane
+        self.tmux_version = self.tmux_pane.display_message("#{version}", True)
+        if width and height:
+            self.resize_window(width, height)
+        self.pause(3)
+        return self.tmux_session
+
+    def start_recording(self, filename: str) -> None:
+        self.asciinema_proc = subprocess.Popen([
+            "asciinema", "rec", "-y", "--overwrite", "-c", "tmux attach -t asciinema_recorder", filename])
+        self.pause(1.5)
+        self.record_start = datetime.datetime.now()
+
+    def resize_window(self, width: int, height: int) -> None:
+        subprocess.Popen(["resize", "-s", str(height), str(width)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    def end(self) -> None:
+        self.end_recording()
+        self.end_session()
+
+    def end_recording(self) -> None:
+        self.asciinema_proc.terminate()
+        self.asciinema_proc.wait(timeout=5)
+
+    def end_session(self) -> None:
+        self.tmux_session.kill_session()
+
+    def press_key(self, keys: str, count=1, pause: typing.Optional[float] = None, target = None) -> None:
+        if pause is None:
+            pause = self.pause_between_keys
+        if target is None:
+            target = self.tmux_pane
+        for i in range(count):
+            if keys == " ":
+                keys = "Space"
+            target.send_keys(cmd=keys, enter=False, suppress_history=False)
+            self.pause(pause + random.uniform(0, self.pause_between_keys_rand))
+
+    def type(self, keys: str, pause: typing.Optional[float] = None, target = None) -> None:
+        if pause is None:
+            pause = self.pause_between_keys
+        if target is None:
+            target = self.tmux_pane
+        target.select_pane()
+        for key in keys:
+            self.press_key(key, pause=pause, target=target)
+
+    def exec(self, keys: str, target = None) -> None:
+        if target is None:
+            target = self.tmux_pane
+        self.type(keys, target=target)
+        self.pause(1.25)
+        self.press_key("Enter", target=target)
+        self.pause(0.5)
+
+    def focus_pane(self, pane: libtmux.Pane, set_active_pane: bool = True) -> None:
+        pane.select_pane()
+        if set_active_pane:
+            self.tmux_pane = pane
+
+    def pause(self, seconds: float) -> None:
+        time.sleep(seconds)
+
+    def run_external(self, command: str) -> None:
+        subprocess.run(command, shell=True)
+
+    def message(self, msg: str, duration: typing.Optional[int] = None, add_instruction: bool = True, instruction_html: str = "") -> None:
+        if duration is None:
+            duration = len(msg) * 0.1 # seconds
+        self.tmux_session.set_option("display-time", int(duration * 1000)) # milliseconds
+        self.tmux_pane.display_message(" " + msg)
+
+        # todo: this is a hack and needs refactoring (instruction() is only defined in MitmCliDirector)
+        if add_instruction or instruction_html:
+            if not instruction_html:
+                instruction_html = msg
+            self.instruction(instruction=instruction_html, duration=duration)
+        self.pause(duration + 0.5)
+
+    def popup(self, content: str, duration: int = 4) -> None:
+        # todo: check if installed tmux version supports display-popup
+
+        # tmux's display-popup is blocking, so we close it in a separate thread
+        t=threading.Thread(target=self.close_popup, args=[duration])
+        t.start()
+
+        lines = content.splitlines()
+        self.tmux_pane.cmd("display-popup", "", *lines)
+        t.join()
+
+    def close_popup(self, duration: float = 0) -> None:
+        self.pause(duration)
+        self.tmux_pane.cmd("display-popup", "-C")
+
+    @property
+    def current_time(self) -> float:
+        now = datetime.datetime.now()
+        return round((now - self.record_start).total_seconds(), 2)
+
+    @property
+    def current_pane(self) -> libtmux.Pane:
+        return self.tmux_pane
+
+
+class InstructionSpec(typing.NamedTuple):
+    instruction: str
+    time_from: float
+    time_from_str: str
+    time_to: float
+
+
+# todo: merge with CliDirector
+class MitmCliDirector(CliDirector):
+    def __init__(self):
+        super().__init__()
+        self.instructions: typing.List[InstructionSpec] = []
+
+    def instruction(self, instruction: str, duration: float = 3, time_from: typing.Optional[float] = None, correction: float = 0) -> None:
+        if time_from is None:
+            time_from = self.current_time
+        time_from_str = str(datetime.timedelta(seconds = int(time_from + correction)))[2:]
+
+        self.instructions.append(InstructionSpec(
+            str(len(self.instructions)+1) + ". " + instruction,
+            time_from=time_from + correction,
+            time_from_str=time_from_str,
+            time_to=time_from - correction + duration
+        ))
+
+    def save_instructions(self, output_path: str) -> None:
+        instr_as_dicts = []
+        for instr in self.instructions:
+            instr_as_dicts.append(instr._asdict())
+        with open(output_path, 'w', encoding='utf-8') as f:
+            json.dump(instr_as_dicts, f, ensure_ascii=False, indent=4)
+
+    def request(self, url: str, threaded: bool = False) -> None:
+        if threaded:
+            threading.Thread(target=lambda: requests.get(url, verify=False)).start()
+        else:
+            requests.get(url, verify=False)
+
+    def end_recording(self) -> None:
+        self.instructions = []
+        super().end_recording()

docs/scripts/clirecording/docker/tmux.conf

@@ -0,0 +1,21 @@
+set -g default-terminal "screen-256color"
+
+set-option -g status-position top
+
+set -g status-style "bg=#000000,fg=#ffffff"
+set -g message-style "bg=#3273dc,fg=#ffffff"
+
+set -g status-justify left
+set -g status-left ""
+set -g status-right ""
+
+setw -g window-status-current-format ""
+
+
+# pane options
+setw -g pane-base-index 1
+setw -g pane-border-format " Terminal Window #P --------------------------------------------------------------------------------------------------------"
+setw -g pane-border-status top
+setw -g pane-border-lines simple
+setw -g pane-border-style "fg=#cccccc"
+setw -g pane-active-border-style "fg=#ffffff"

docs/scripts/clirecording/generate_recordings.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+docker build --pull --rm -t mitmproxy-clirecorder:latest .
+docker run -i -t --rm \
+    -v "$(pwd)"/../../src/static/recordings:/root/clidirector/recordings \
+    mitmproxy-clirecorder:latest

docs/scripts/clirecording/record.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+from clidirector import MitmCliDirector
+import screenplays
+
+
+if __name__ == '__main__':
+    director = MitmCliDirector()
+    screenplays.record_user_interface(director)
+    screenplays.record_intercept_requests(director)
+    screenplays.record_modify_requests(director)
+    screenplays.record_replay_requests(director)

docs/scripts/clirecording/screenplays.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+
+from clidirector import MitmCliDirector
+
+
+def record_user_interface(d: MitmCliDirector):
+    tmux = d.start_session(width=120, height=36)
+    window = tmux.attached_window
+
+    d.start_recording("recordings/mitmproxy_user_interface.cast")
+    d.message("Welcome to the mitmproxy tutorial. In this lesson we cover the user interface.")
+    d.pause(1)
+    d.exec("mitmproxy")
+    d.pause(3)
+
+    d.message("This is the default view of mitmproxy.")
+    d.message("mitmproxy adds rows to the view as new requests come in.")
+    d.message("Let’s generate some requests using `curl` in a separate terminal.")
+
+    pane_top = d.current_pane
+    pane_bottom = window.split_window(attach=True)
+    pane_bottom.resize_pane(height=12)
+
+    d.focus_pane(pane_bottom)
+    d.pause(2)
+
+    d.type("curl")
+    d.message("Use curl’s `-x` option to specify a proxy, e.g., `curl -x http://127.0.0.1:8080` to use mitmproxy.")
+
+    d.type(" -x http://127.0.0.1:8080")
+
+    d.message("We use the text-based weather service `wttr.in`.")
+    d.exec(" \"http://wttr.in/Paris?0\"")
+
+    d.pause(2)
+    d.press_key("Up")
+    d.press_key("Left", count=3)
+    d.press_key("BSpace", count=5)
+    d.exec("Miami")
+
+    d.pause(2)
+    d.press_key("Up")
+    d.press_key("Left", count=3)
+    d.press_key("BSpace", count=5)
+    d.exec("Tokio")
+
+    d.pause(2)
+    d.press_key("Up")
+    d.press_key("Left", count=3)
+    d.press_key("BSpace", count=5)
+    d.exec("London")
+
+    d.pause(2)
+    d.exec("exit", target=pane_bottom)
+
+    d.focus_pane(pane_top)
+
+    d.message("You see the requests to `wttr.in` in the list of flows.")
+
+    d.message("mitmproxy is controlled using keyboard shortcuts.")
+    d.message("Use your arrow keys `↑` and `↓` to change the focused flow (`>>`).")
+    d.press_key("Down", count=3, pause=0.5)
+    d.press_key("Up", count=2, pause=0.5)
+    d.press_key("Down", count=2, pause=0.5)
+
+    d.message("The focused flow (`>>`) is used as a target for various commands.")
+
+    d.message("One such command shows the flow details, it is bound to `↵`.")
+
+    d.message("Press `↵` to view the details of the focused flow.")
+    d.press_key("Enter")
+
+    d.message("The flow details view has 3 panes: request, response, and detail.")
+    d.message("Use your arrow keys `←` and `→` to switch between panes.")
+    d.press_key("Right", count=2, pause=2.5)
+    d.press_key("Left", count=2, pause=1)
+
+    d.message("Press `q` to exit the current view.",)
+    d.type("q")
+
+    d.message("Press `?` to get a list of all available keyboard shortcuts.")
+    d.type("?")
+    d.pause(2)
+    d.press_key("Down", count=20, pause=0.25)
+
+    d.message("Press `q` to exit the current view.")
+    d.type("q")
+
+    d.message("Each shortcut is internally bound to a command.")
+    d.message("You can also execute commands directly (without using shortcuts).")
+    d.message("Press `:` to open the command prompt at the bottom.")
+    d.type(":")
+
+    d.message("Enter `console.view.flow @focus`.")
+    d.type("console.view.flow @focus")
+
+    d.message("The command `console.view.flow` opens the details view for a flow.")
+
+    d.message("The argument `@focus` defines the target flow.")
+
+    d.message("Press `↵` to execute the command.")
+    d.press_key("Enter")
+
+    d.message("Commands unleash the full power of mitmproxy, i.e., to configure interceptions.")
+
+    d.message("You now know basics of mitmproxy’s UI and how to control it.")
+    d.pause(1)
+    d.save_instructions("recordings/mitmproxy_user_interface_instructions.json")
+    d.end()
+
+
+def record_intercept_requests(d: MitmCliDirector):
+    tmux = d.start_session(width=120, height=36)
+    window = tmux.attached_window
+
+    d.start_recording("recordings/mitmproxy_intercept_requests.cast")
+    d.message("Welcome to the mitmproxy tutorial. In this lesson we cover the interception of requests.")
+    d.pause(1)
+    d.exec("mitmproxy")
+    d.pause(3)
+
+    d.message("We first need to configure mitmproxy to intercept requests.")
+
+    d.message("Press `i` to prepopulate mitmproxy’s command prompt with `set intercept ''`.")
+    d.type("i")
+    d.pause(2)
+
+    d.message("We use the flow filter expression `~u <regex>` to only intercept specific URLs.")
+    d.message("Additionally, we use the filter `~q` to only intercept requests, but not responses.")
+    d.message("We combine both flow filters using `&`.")
+
+    d.message("Enter `~u /Paris & ~q` between the quotes of the `set intercept` command and press `↵`.")
+    d.exec("~u /Paris & ~q")
+    d.message("The bottom bar shows that the interception has been configured.")
+
+    d.message("Let’s generate a request using `curl` in a separate terminal.")
+
+    pane_top = d.current_pane
+    pane_bottom = window.split_window(attach=True)
+    pane_bottom.resize_pane(height=12)
+
+    d.focus_pane(pane_bottom)
+    d.pause(2)
+
+    d.type("curl")
+    d.message("Use curl’s `-x` option to specify a proxy, e.g., `curl -x http://127.0.0.1:8080` to use mitmproxy.")
+
+    d.type(" -x http://127.0.0.1:8080")
+
+    d.message("We use the text-based weather service `wttr.in`.")
+    d.exec(" \"http://wttr.in/Paris?0\"")
+    d.pause(2)
+
+    d.focus_pane(pane_top)
+
+    d.message("You will see a new line in in the list of flows.")
+    d.message("The new flow is displayed in red to indicate that it has been intercepted.")
+    d.message("Put the focus (`>>`) on the intercepted flow. This is already the case in our example.")
+    d.message("Press `a` to resume this flow without making any changes.")
+    d.type("a")
+
+    d.focus_pane(pane_bottom)
+
+    d.message("Submit another request and focus its flow.")
+    d.press_key("Up")
+    d.press_key("Enter")
+    d.pause(2)
+
+    d.focus_pane(pane_top)
+    d.press_key("Down")
+
+    d.message("Press `X` to kill this flow, i.e., discard it without forwarding it to its final destination `wttr.in`.")
+    d.type("X")
+    d.pause(3)
+    d.save_instructions("recordings/mitmproxy_intercept_requests_instructions.json")
+    d.end()
+
+
+def record_modify_requests(d: MitmCliDirector):
+    tmux = d.start_session(width=120, height=36)
+    window = tmux.attached_window
+
+    d.start_recording("recordings/mitmproxy_modify_requests.cast")
+    d.message("Welcome to the mitmproxy tutorial. In this lesson we cover the modification of intercepted requests.")
+    d.pause(1)
+    d.exec("mitmproxy")
+    d.pause(3)
+
+    d.message("We configure and use the same interception rule as in the last tutorial.")
+    d.message("Press `i` to prepopulate mitmproxy’s command prompt, enter the flow filter `~u /Paris & ~q`, and press `↵`.")
+    d.type("i")
+    d.pause(2)
+    d.exec("~u /Paris & ~q")
+
+    d.message("Let’s generate a request using `curl` in a separate terminal.")
+
+    pane_top = d.current_pane
+    pane_bottom = window.split_window(attach=True)
+    pane_bottom.resize_pane(height=12)
+
+    d.focus_pane(pane_bottom)
+    d.pause(2)
+
+    d.type("curl -x http://127.0.0.1:8080")
+    d.exec(" \"http://wttr.in/Paris?0\"")
+    d.pause(2)
+
+    d.focus_pane(pane_top)
+
+    d.message("We now want to modify the intercepted request.")
+    d.message("Put the focus (`>>`) on the intercepted flow. This is already the case in our example.")
+
+    d.message("Press `↵` to open the details view for the intercepted flow.")
+    d.press_key("Enter")
+
+    d.message("Press `e` to edit the intercepted flow.")
+    d.type("e")
+
+    d.message("mitmproxy asks which part to modify.")
+
+    d.message("Select `path` by using your arrow keys and press `↵`.")
+    d.press_key("Down", count=3, pause=0.5)
+    d.pause(1)
+    d.press_key("Enter")
+
+    d.message("mitmproxy shows all path components line by line, in our example its just one element: `Paris`.")
+    d.message("Press `↵` to modify the selected path component.")
+    d.press_key("Down", pause=2)
+    d.press_key("Enter")
+
+    d.message("Replace `Paris` with `Tokio`.")
+    d.press_key("BSpace", count=5, pause=0.5)
+    d.type("Tokio", pause=0.5)
+
+    d.message("Press `ESC` to confirm your change.")
+    d.press_key("Escape")
+
+    d.message("Press `q` to go back to the flow details view.")
+    d.type("q")
+
+    d.message("Press `a` to resume the intercepted flow.")
+    d.type("a")
+    d.pause(2)
+
+    d.message("You see that the request URL was modified and `wttr.in` replied with the weather report for `Tokio`.")
+
+    d.save_instructions("recordings/mitmproxy_modify_requests_instructions.json")
+    d.end()
+
+
+def record_replay_requests(d: MitmCliDirector):
+    tmux = d.start_session(width=120, height=36)
+    window = tmux.attached_window
+
+    d.start_recording("recordings/mitmproxy_replay_requests.cast")
+    d.message("Welcome to the mitmproxy tutorial. In this lesson we cover replaying requests.")
+    d.pause(1)
+    d.exec("mitmproxy")
+    d.pause(3)
+
+    d.message("Let’s generate a request that we can replay. We use `curl` in a separate terminal.")
+
+    pane_top = d.current_pane
+    pane_bottom = window.split_window(attach=True)
+    pane_bottom.resize_pane(height=12)
+
+    d.focus_pane(pane_bottom)
+    d.pause(2)
+
+    d.exec("curl -x http://127.0.0.1:8080 \"http://wttr.in/Paris?0\"")
+    d.pause(2)
+
+    d.focus_pane(pane_top)
+
+    d.message("We now want to replay the intercepted request.")
+    d.message("Put the focus (`>>`) on the intercepted flow. This is already the case in our example.")
+    d.message("Press `r` to replay this flow.")
+    d.type("r")
+
+    d.message("Note that no new rows are added for replayed flows, but the existing row is updated.")
+    d.message("Every time you press `r`, mitmproxy sends this request to the server again and updates the flow.")
+    d.press_key("r", count=6, pause=0.75)
+
+    d.message("You can also modify a flow before replaying it.")
+    d.message("It works as shown in the previous tutorial by pressing `e`.")
+
+    d.save_instructions("recordings/mitmproxy_replay_requests_instructions.json")
+    d.end()