[sans-io] enable proxying by host header

This mirrors the current examples/complex/dns_spoofing.py script.

mitmproxy/proxy2/layers/http/__init__.py

@@ -1,10 +1,11 @@
 import collections
+import re
 import typing
 from dataclasses import dataclass
 
-from mitmproxy import exceptions, flow, http
+from mitmproxy import flow, http
 from mitmproxy.net import server_spec
-from mitmproxy.proxy.protocol.http import HTTPMode, validate_request_form
+from mitmproxy.proxy.protocol.http import HTTPMode
 from mitmproxy.proxy2 import commands, events, layer, tunnel
 from mitmproxy.proxy2.context import Connection, Context, Server
 from mitmproxy.proxy2.layers import tls
@@ -19,6 +20,28 @@ from ._hooks import HttpConnectHook, HttpErrorHook, HttpRequestHeadersHook, Http
 from ._http1 import Http1Client, Http1Server
 from ._http2 import Http2Client
 
+# This regex extracts splits the host header into host and port.
+# Handles the edge case of IPv6 addresses containing colons.
+# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
+parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
+
+
+def validate_request(mode, request) -> typing.Optional[str]:
+    if request.scheme not in ("http", None):
+        return f"Invalid request scheme: {request.scheme}"
+    if mode is HTTPMode.transparent:
+        if request.first_line_format == "authority":
+            return (
+                f"mitmproxy received an HTTP CONNECT request even though it is not running in regular/upstream mode. "
+                f"This usually indicates a misconfiguration, please see the mitmproxy mode documentation for details."
+            )
+        if request.first_line_format == "absolute":
+            return (
+                f"mitmproxy received an absolute-form HTTP request even though it is not running in regular/upstream mode. "
+                f"This usually indicates a misconfiguration, please see the mitmproxy mode documentation for details."
+            )
+    return None
+
 
 @dataclass(unsafe_hash=True)
 class GetHttpConnection(HttpCommand):
@@ -119,23 +142,40 @@ class HttpStream(layer.Layer):
             # self.send_response(http.expect_continue_response)
             # request.headers.pop("expect")
 
-        try:
+        if err := validate_request(self.mode, self.flow.request):
-            validate_request_form(self.mode, self.flow.request)
+            self.flow.response = http.HTTPResponse.make(502, str(err))
-        except exceptions.HttpException as e:
+            self.client_state = self.state_errored
-            self.flow.response = http.HTTPResponse.make(502, str(e))
             return (yield from self.send_response())
 
+        if self.flow.request.first_line_format == "authority":
+            return (yield from self.handle_connect())
+
+        if self.mode is HTTPMode.regular and self.flow.request.first_line_format == "absolute":
             # set first line format to relative in regular mode,
             # see https://github.com/mitmproxy/mitmproxy/issues/1759
-        if self.mode is HTTPMode.regular and self.flow.request.first_line_format == "absolute":
             self.flow.request.first_line_format = "relative"
-
+        elif self.mode in (HTTPMode.regular, HTTPMode.upstream) and self.flow.request.first_line_format == "relative":
-        if self.mode is HTTPMode.upstream:
+            # We need to extract destination information from the host header.
-            self.context.server.via = server_spec.parse_with_mode(self.context.options.mode)[1]
+            if m := parse_host_header.match(self.flow.request.host_header or ""):
-
+                host = m.group("host").strip("[]")
-        # Determine .scheme, .host and .port attributes for relative-form requests
+                if m.group("port"):
-        if self.mode is HTTPMode.transparent:
+                    port = int(m.group("port"))
-            # Setting request.host also updates the host header, which we want to preserve
+                else:
+                    port = 443 if self.context.client.tls else 80
+                host_header = self.flow.request.host_header
+                self.flow.request.host = host
+                self.flow.request.port = port
+                self.flow.request.host_header = host_header  # set again as .host overwrites this.
+                self.flow.request.scheme = "https" if self.context.client.tls else "http"
+            else:
+                self.flow.response = http.HTTPResponse.make(
+                    400,
+                    "HTTP request has no host header, destination unknown."
+                )
+                self.client_state = self.state_errored
+                return (yield from self.send_response())
+        elif self.mode is HTTPMode.transparent:
+            # Determine .scheme, .host and .port attributes for transparent requests
             host_header = self.flow.request.host_header
             self.flow.request.host = self.context.server.address[0]
             self.flow.request.port = self.context.server.address[1]
@@ -146,9 +186,6 @@ class HttpStream(layer.Layer):
         if self.context.options.mode.startswith("reverse:") and not self.context.options.keep_host_header:
             self.flow.request.host_header = self.context.server.address[0]
 
-        if self.flow.request.first_line_format == "authority":
-            return (yield from self.handle_connect())
-
         yield HttpRequestHeadersHook(self.flow)
 
         if self.flow.request.stream:
@@ -230,13 +267,13 @@ class HttpStream(layer.Layer):
             self.flow.response.data.content = self.response_body_buf
             self.response_body_buf = b""
             yield from self.send_response()
+            self.server_state = self.state_done
 
     def send_response(self):
         yield HttpResponseHook(self.flow)
         yield SendHttp(ResponseHeaders(self.stream_id, self.flow.response), self.context.client)
         yield SendHttp(ResponseData(self.stream_id, self.flow.response.data.content), self.context.client)
         yield SendHttp(ResponseEndOfMessage(self.stream_id), self.context.client)
-        self.server_state = self.state_done
 
     def handle_protocol_error(
             self,
@@ -313,6 +350,7 @@ class HttpStream(layer.Layer):
                     http_proxy.child_layer = layer.NextLayer(self.context)
                     yield from http_proxy.child_layer.handle_event(events.Start())
                     self._handle_event = self.passthrough
+
             self._handle_event = _wait_for_reply
         else:
             self.child_layer = http_proxy
@@ -395,7 +433,8 @@ class HttpLayer(layer.Layer):
 
     def _handle_event(self, event: events.Event):
         if isinstance(event, events.Start):
-            return
+            if self.mode is HTTPMode.upstream:
+                self.context.server.via = server_spec.parse_with_mode(self.context.options.mode)[1]
         elif isinstance(event, events.CommandReply):
             stream = self.stream_by_command.pop(event.command)
             self.event_to_child(stream, event)

mitmproxy/proxy2/server.py

@@ -253,7 +253,7 @@ if __name__ == "__main__":
 
     opts = moptions.Options()
     opts.add_option(
-        "connection_strategy", str, "eager",
+        "connection_strategy", str, "lazy",
         "Determine when server connections should be established.",
         choices=("eager", "lazy")
     )

test/mitmproxy/proxy2/layers/test_http.py

@@ -5,8 +5,8 @@ from mitmproxy.proxy.protocol.http import HTTPMode
 from mitmproxy.proxy2 import layer
 from mitmproxy.proxy2.commands import CloseConnection, OpenConnection, SendData
 from mitmproxy.proxy2.events import ConnectionClosed, DataReceived
-from mitmproxy.proxy2.layers import http, tls, TCPLayer
+from mitmproxy.proxy2.layers import TCPLayer, http, tls
-from test.mitmproxy.proxy2.tutils import Placeholder, Playbook, reply, reply_next_layer, EchoLayer
+from test.mitmproxy.proxy2.tutils import Placeholder, Playbook, reply, reply_next_layer
 
 
 def test_http_proxy(tctx):
@@ -510,11 +510,10 @@ def test_proxy_chain(tctx, strategy):
     playbook >> reply_next_layer(lambda ctx: http.HttpLayer(ctx, HTTPMode.transparent))
     playbook << SendData(tctx.client,
                          b"HTTP/1.1 502 Bad Gateway\r\n"
-                         b"content-length: 189\r\n"
+                         b"content-length: 198\r\n"
                          b"\r\n"
-                         b"Mitmproxy received an HTTP CONNECT request even though it is not running\n"
+                         b"mitmproxy received an HTTP CONNECT request even though it is not running in regular/upstream mode. "
-                         b"in regular mode. This usually indicates a misconfiguration,\n"
+                         b"This usually indicates a misconfiguration, please see the mitmproxy mode documentation for details.")
-                         b"please see the mitmproxy mode documentation for details.")
 
     assert playbook
 
@@ -535,9 +534,8 @@ def test_no_headers(tctx):
     assert server().address == ("example.com", 80)
 
 
-@pytest.mark.xfail
 def test_http_proxy_relative_request(tctx):
-    """Test handling of a relative-form "GET /"."""
+    """Test handling of a relative-form "GET /" in regular proxy mode."""
     server = Placeholder()
     assert (
             Playbook(http.HttpLayer(tctx, HTTPMode.regular), hooks=False)
@@ -549,3 +547,15 @@ def test_http_proxy_relative_request(tctx):
             << SendData(tctx.client, b"HTTP/1.1 204 No Content\r\n\r\n")
     )
     assert server().address == ("example.com", 80)
+
+
+def test_http_proxy_relative_request_no_host_header(tctx):
+    """Test handling of a relative-form "GET /" in regular proxy mode, but without a host header."""
+    assert (
+            Playbook(http.HttpLayer(tctx, HTTPMode.regular), hooks=False)
+            >> DataReceived(tctx.client, b"GET / HTTP/1.1\r\n\r\n")
+            << SendData(tctx.client, b"HTTP/1.1 400 Bad Request\r\n"
+                                     b"content-length: 53\r\n"
+                                     b"\r\n"
+                                     b"HTTP request has no host header, destination unknown.")
+    )