Merge pull request #4384 from mhils/mitmproxy.exe

mitmproxy.exe 🎉🥳

CHANGELOG.md

@@ -38,6 +38,7 @@ If you depend on these features, please raise your voice in
 ### Full Changelog
 
 * New Proxy Core based on sans-io pattern (@mhils)
+* mitmproxy's command line interface now supports Windows (@mhils)
 * Use pyca/cryptography to generate certificates, not pyOpenSSL (@mhils)
 * Remove the legacy protocol stack (@Kriechi)
 * Remove all deprecated pathod and pathoc tools and modules (@Kriechi)

docs/src/content/overview-installation.md

@@ -34,17 +34,15 @@ the repository maintainers directly for issues with native packages.
 
 ## Windows
 
-All the mitmproxy tools are fully supported under [WSL (Windows Subsystem for
-Linux)](https://docs.microsoft.com/en-us/windows/wsl/about). We recommend to
-[install WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10), and
-then follow the mitmproxy installation instructions for Linux.
+To install mitmproxy on Windows, download the installer from [mitmproxy.org](https://mitmproxy.org/). After
+installation, mitmproxy, mitmdump and mitmweb are also added to your PATH and can be invoked from the command line.
 
-We also distribute native Windows packages for all tools other than the
-mitmproxy console app, which only works under WSL. To install mitmproxy on
-Windows, download the installer from [mitmproxy.org](https://mitmproxy.org/).
-After installation, you'll find shortcuts for mitmweb and mitmdump in the start
-menu. Both executables are added to your PATH and can be invoked from the
-command line.
+We highly recommend to [install Windows Terminal](https://aka.ms/terminal) to improve the rendering of the console interface.
+
+All the mitmproxy tools are also supported under
+[WSL (Windows Subsystem for Linux)](https://docs.microsoft.com/en-us/windows/wsl/about). After
+[installing WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10), follow the mitmproxy installation
+instructions for Linux.
 
 ## Advanced Installation
 
@@ -80,9 +78,8 @@ You can use the official mitmproxy images from
 ### Security Considerations for Binary Packages
 
 Our pre-compiled binary packages and Docker images include a self-contained
-Python 3 environment,  a recent version of OpenSSL that support ALPN and HTTP/2,
-and other dependencies that would otherwise be cumbersome to compile and
-install.
+Python 3 environment, a recent version of OpenSSL, and other dependencies
+that would otherwise be cumbersome to compile and install.
 
 Dependencies in the binary packages are frozen on release, and can't be updated
 in situ. This means that we necessarily capture any bugs or security issues that

mitmproxy/contrib/README

@@ -4,6 +4,5 @@ Contribs:
 wbxml
     - https://github.com/davidpshaw/PyWBXMLDecoder
 
-tls, BSD license
-    - https://github.com/mhils/tls/tree/mitmproxy
-    - limited to required files.
+urwid
+    - Patches vendored from https://github.com/urwid/urwid/pull/448.

mitmproxy/contrib/urwid/win32.py

@@ -0,0 +1,148 @@
+from ctypes import Structure, Union, windll, POINTER
+from ctypes.wintypes import BOOL, DWORD, WCHAR, WORD, SHORT, UINT, HANDLE, LPDWORD, CHAR
+
+# https://docs.microsoft.com/de-de/windows/console/getstdhandle
+STD_INPUT_HANDLE = -10
+STD_OUTPUT_HANDLE = -11
+
+# https://docs.microsoft.com/de-de/windows/console/setconsolemode
+ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004
+DISABLE_NEWLINE_AUTO_RETURN = 0x0008
+ENABLE_VIRTUAL_TERMINAL_INPUT = 0x0200
+ENABLE_WINDOW_INPUT = 0x0008
+
+
+class COORD(Structure):
+    """https://docs.microsoft.com/en-us/windows/console/coord-str"""
+
+    _fields_ = [
+        ("X", SHORT),
+        ("Y", SHORT),
+    ]
+
+
+class SMALL_RECT(Structure):
+    """https://docs.microsoft.com/en-us/windows/console/small-rect-str"""
+
+    _fields_ = [
+        ("Left", SHORT),
+        ("Top", SHORT),
+        ("Right", SHORT),
+        ("Bottom", SHORT),
+    ]
+
+
+class CONSOLE_SCREEN_BUFFER_INFO(Structure):
+    """https://docs.microsoft.com/en-us/windows/console/console-screen-buffer-info-str"""
+
+    _fields_ = [
+        ("dwSize", COORD),
+        ("dwCursorPosition", COORD),
+        ("wAttributes", WORD),
+        ("srWindow", SMALL_RECT),
+        ("dwMaximumWindowSize", COORD),
+    ]
+
+
+class uChar(Union):
+    """https://docs.microsoft.com/en-us/windows/console/key-event-record-str"""
+    _fields_ = [
+        ("AsciiChar", CHAR),
+        ("UnicodeChar", WCHAR),
+    ]
+
+
+class KEY_EVENT_RECORD(Structure):
+    """https://docs.microsoft.com/en-us/windows/console/key-event-record-str"""
+
+    _fields_ = [
+        ("bKeyDown", BOOL),
+        ("wRepeatCount", WORD),
+        ("wVirtualKeyCode", WORD),
+        ("wVirtualScanCode", WORD),
+        ("uChar", uChar),
+        ("dwControlKeyState", DWORD),
+    ]
+
+
+class MOUSE_EVENT_RECORD(Structure):
+    """https://docs.microsoft.com/en-us/windows/console/mouse-event-record-str"""
+
+    _fields_ = [
+        ("dwMousePosition", COORD),
+        ("dwButtonState", DWORD),
+        ("dwControlKeyState", DWORD),
+        ("dwEventFlags", DWORD),
+    ]
+
+
+class WINDOW_BUFFER_SIZE_RECORD(Structure):
+    """https://docs.microsoft.com/en-us/windows/console/window-buffer-size-record-str"""
+
+    _fields_ = [("dwSize", COORD)]
+
+
+class MENU_EVENT_RECORD(Structure):
+    """https://docs.microsoft.com/en-us/windows/console/menu-event-record-str"""
+
+    _fields_ = [("dwCommandId", UINT)]
+
+
+class FOCUS_EVENT_RECORD(Structure):
+    """https://docs.microsoft.com/en-us/windows/console/focus-event-record-str"""
+
+    _fields_ = [("bSetFocus", BOOL)]
+
+
+class Event(Union):
+    """https://docs.microsoft.com/en-us/windows/console/input-record-str"""
+    _fields_ = [
+        ("KeyEvent", KEY_EVENT_RECORD),
+        ("MouseEvent", MOUSE_EVENT_RECORD),
+        ("WindowBufferSizeEvent", WINDOW_BUFFER_SIZE_RECORD),
+        ("MenuEvent", MENU_EVENT_RECORD),
+        ("FocusEvent", FOCUS_EVENT_RECORD),
+    ]
+
+
+class INPUT_RECORD(Structure):
+    """https://docs.microsoft.com/en-us/windows/console/input-record-str"""
+
+    _fields_ = [
+        ("EventType", WORD),
+        ("Event", Event)
+    ]
+
+
+class EventType:
+    FOCUS_EVENT = 0x0010
+    KEY_EVENT = 0x0001
+    MENU_EVENT = 0x0008
+    MOUSE_EVENT = 0x0002
+    WINDOW_BUFFER_SIZE_EVENT = 0x0004
+
+
+# https://docs.microsoft.com/de-de/windows/console/getstdhandle
+GetStdHandle = windll.kernel32.GetStdHandle
+GetStdHandle.argtypes = [DWORD]
+GetStdHandle.restype = HANDLE
+
+# https://docs.microsoft.com/de-de/windows/console/getconsolemode
+GetConsoleMode = windll.kernel32.GetConsoleMode
+GetConsoleMode.argtypes = [HANDLE, LPDWORD]
+GetConsoleMode.restype = BOOL
+
+# https://docs.microsoft.com/de-de/windows/console/setconsolemode
+SetConsoleMode = windll.kernel32.SetConsoleMode
+SetConsoleMode.argtypes = [HANDLE, DWORD]
+SetConsoleMode.restype = BOOL
+
+# https://docs.microsoft.com/de-de/windows/console/readconsoleinput
+ReadConsoleInputW = windll.kernel32.ReadConsoleInputW
+# ReadConsoleInputW.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, LPDWORD]
+ReadConsoleInputW.restype = BOOL
+
+# https://docs.microsoft.com/en-us/windows/console/getconsolescreenbufferinfo
+GetConsoleScreenBufferInfo = windll.kernel32.GetConsoleScreenBufferInfo
+GetConsoleScreenBufferInfo.argtypes = [HANDLE, POINTER(CONSOLE_SCREEN_BUFFER_INFO)]
+GetConsoleScreenBufferInfo.restype = BOOL

mitmproxy/tools/console/common.py

@@ -13,8 +13,8 @@ from mitmproxy.http import HTTPFlow
 from mitmproxy.utils import human
 from mitmproxy.tcp import TCPFlow
 
-# Detect Windows Subsystem for Linux
-IS_WSL = "Microsoft" in platform.platform()
+# Detect Windows Subsystem for Linux and Windows
+IS_WINDOWS = "Microsoft" in platform.platform() or "Windows" in platform.platform()
 
 
 def is_keypress(k):

mitmproxy/tools/console/window.py

@@ -1,3 +1,4 @@
+import os
 import re
 
 import urwid
@@ -14,6 +15,11 @@ from mitmproxy.tools.console import overlay
 from mitmproxy.tools.console import signals
 from mitmproxy.tools.console import statusbar
 
+if os.name == "nt":
+    from mitmproxy.contrib.urwid import raw_display
+else:
+    from urwid import raw_display
+
 
 class StackWidget(urwid.Frame):
     def __init__(self, window, widget, title, focus):
@@ -315,10 +321,10 @@ class Window(urwid.Frame):
             )
 
 
-class Screen(urwid.raw_display.Screen):
+class Screen(raw_display.Screen):
 
     def write(self, data):
-        if common.IS_WSL:
+        if common.IS_WINDOWS:
             # replace urwid's SI/SO, which produce artifacts under WSL.
             # at some point we may figure out what they actually do.
             data = re.sub("[\x0e\x0f]", "", data)

mitmproxy/tools/main.py

@@ -122,10 +122,10 @@ def run(
 
 def mitmproxy(args=None) -> typing.Optional[int]:  # pragma: no cover
     if os.name == "nt":
-        print("Error: mitmproxy's console interface is not supported on Windows. "
-              "You can run mitmdump or mitmweb instead.", file=sys.stderr)
-        return 1
-    assert_utf8_env()
+        import urwid
+        urwid.set_encoding("utf8")
+    else:
+        assert_utf8_env()
     from mitmproxy.tools import console
     run(console.master.ConsoleMaster, cmdline.mitmproxy, args)
     return None

release/cibuild.py

@@ -121,12 +121,9 @@ class BuildEnviron:
 
     @property
     def bdists(self):
-        ret = {
+        return {
             "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
         }
-        if self.system == "Windows":
-            ret["mitmproxy"].remove("mitmproxy")
-        return ret
 
     @property
     def branch(self) -> str:

release/installbuilder/mitmproxy.xml

@@ -33,6 +33,9 @@
                             <allowWildcards>1</allowWildcards>
                             <origin>../build/binaries/${platform_name}/*</origin>
                         </distributionFile>
+                        <distributionFile>
+                            <origin>run.ps1</origin>
+                        </distributionFile>
                     </distributionFileList>
                 </folder>
             </folderList>
@@ -51,24 +54,34 @@
             </postUninstallationActionList>
             <startMenuShortcutList>
                 <startMenuShortcut>
-                    <comment></comment>
-                    <name>mitmproxy ui</name>
+                    <comment>mitmproxy command line interface</comment>
+                    <name>mitmproxy</name>
                     <runAsAdmin>0</runAsAdmin>
                     <runInTerminal>0</runInTerminal>
-                    <windowsExec>${installdir}\bin\mitmweb.exe</windowsExec>
-                    <windowsExecArgs></windowsExecArgs>
+                    <windowsExec>powershell.exe</windowsExec>
+                    <windowsExecArgs>-File "${installdir}\bin\run.ps1" mitmproxy</windowsExecArgs>
                     <windowsIcon>${installdir}/logo.ico</windowsIcon>
-                    <windowsPath>${installdir}</windowsPath>
+                    <windowsPath>${user_home_directory}</windowsPath>
                 </startMenuShortcut>
                 <startMenuShortcut>
-                    <comment></comment>
+                    <comment>mitmproxy web interface</comment>
+                    <name>mitmweb</name>
+                    <runAsAdmin>0</runAsAdmin>
+                    <runInTerminal>0</runInTerminal>
+                    <windowsExec>powershell.exe</windowsExec>
+                    <windowsExecArgs>-File "${installdir}\bin\run.ps1" mitmweb</windowsExecArgs>
+                    <windowsIcon>${installdir}/logo.ico</windowsIcon>
+                    <windowsPath>${user_home_directory}</windowsPath>
+                </startMenuShortcut>
+                <startMenuShortcut>
+                    <comment>mitmproxy non-interactive interface</comment>
                     <name>mitmdump</name>
                     <runAsAdmin>0</runAsAdmin>
                     <runInTerminal>0</runInTerminal>
-                    <windowsExec>${installdir}\bin\mitmdump.exe</windowsExec>
-                    <windowsExecArgs></windowsExecArgs>
+                    <windowsExec>powershell.exe</windowsExec>
+                    <windowsExecArgs>-File "${installdir}\bin\run.ps1" mitmdump</windowsExecArgs>
                     <windowsIcon>${installdir}/logo.ico</windowsIcon>
-                    <windowsPath>${installdir}</windowsPath>
+                    <windowsPath>${user_home_directory}</windowsPath>
                 </startMenuShortcut>
             </startMenuShortcutList>
         </component>
@@ -85,24 +98,9 @@
     <finalPageActionList>
         <runProgram>
             <program>cmd</program>
-            <programArguments>/c start "mitmproxy ui" "${installdir}\bin\mitmweb.exe" &amp;</programArguments>
-            <progressText>Launch mitmproxy ui now</progressText>
-            <ruleList>
-                <platformTest>
-                    <type>windows</type>
-                </platformTest>
-            </ruleList>
-        </runProgram>
-        <runProgram>
-            <program>${installdir}/mitmproxy</program>
-            <programArguments>&amp;</programArguments>
+            <programArguments>/c powershell -File "${installdir}\bin\run.ps1" mitmproxy &amp;</programArguments>
             <progressText>Launch mitmproxy now</progressText>
-            <ruleList>
-                <platformTest>
-                    <negate>1</negate>
-                    <type>windows</type>
-                </platformTest>
-            </ruleList>
+            <workingDirectory>${user_home_directory}</workingDirectory>
         </runProgram>
     </finalPageActionList>
     <parameterList>
@@ -120,6 +118,7 @@
             <width>40</width>
             <postShowPageActionList>
                 <!-- This will skip the readytoinstall page -->
+
                 <setInstallerVariable name="next_page" value="installation"/>
             </postShowPageActionList>
             <preShowPageActionList>
@@ -131,3 +130,4 @@
         </directoryParameter>
     </parameterList>
 </project>
+

release/installbuilder/run.ps1

@@ -0,0 +1,5 @@
+if (Get-Command wt -ErrorAction SilentlyContinue) { 
+	Start-Process wt -ArgumentList "powershell.exe","-NoExit","-Command",$args[0]
+} else { 
+	Start-Process powershell -ArgumentList "-NoExit","-Command",$args[0]
+}

test/mitmproxy/tools/console/test_integration.py

@@ -8,7 +8,6 @@ import mitmproxy.options
 from mitmproxy import master
 from mitmproxy.tools.console import window
 from mitmproxy.tools.console.master import ConsoleMaster
-from test.conftest import skip_windows
 
 
 def tokenize(input: str) -> List[str]:
@@ -40,7 +39,6 @@ def console(monkeypatch):
     return m
 
 
-@skip_windows
 def test_integration(tdata, console):
     console.type(f":view.flows.load {tdata.path('mitmproxy/data/dumpfile-018.bin')}<enter>")
     console.type("<enter><tab><tab>")

test/release/test_cibuild.py

@@ -205,7 +205,7 @@ def test_buildenviron_windows(tmpdir):
     )
     assert be.platform_tag == "windows"
     assert be.bdists == {
-        "mitmproxy": ["mitmdump", "mitmweb"],
+        "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
     }
     assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-windows.zip"