Merge pull request #4225 from bburky/send-http1-trailers

Send http/1 request and response trailer headers

CHANGELOG

@@ -18,6 +18,7 @@ Unreleased: mitmproxy next
     * Add interactive tutorials to the documentation (@mplattner)
     * Support `deflateRaw` `Content-Encoding`s (@kjoconnor)
     * Fix broken requests without body on HTTP/2 (@Kriechi)
+    * Add support for sending (but not parsing) HTTP Trailers to the HTTP/1.1 protocol (@bburky)
 
     * --- TODO: add new PRs above this line ---
 

docs/src/content/concepts-protocols.md

@@ -16,7 +16,7 @@ menu:
 HTTP/1.0 and HTTP/1.1 support in mitmproxy is based on our custom HTTP stack,
 which takes care of all semantics and on-the-wire parsing/serialization tasks.
 
-mitmproxy currently does not support HTTP trailers - but if you want to send
+mitmproxy currently does not support parsing HTTP trailers - but if you want to send
 us a PR, we promise to take look!
 
 ## HTTP/2

examples/addons/http-trailers.py

@@ -15,14 +15,40 @@ def request(flow: http.HTTPFlow):
     if flow.request.trailers:
         print("HTTP Trailers detected! Request contains:", flow.request.trailers)
 
+    if flow.request.path == "/inject_trailers":
+        if flow.request.is_http10:
+            # HTTP/1.0 doesn't support trailers
+            return
+        elif flow.request.is_http11:
+            if not flow.request.content:
+                # Avoid sending a body on GET requests or a 0 byte chunked body with trailers.
+                # Otherwise some servers return 400 Bad Request.
+                return
+            # HTTP 1.1 requires transfer-encoding: chunked to send trailers
+            flow.request.headers["transfer-encoding"] = "chunked"
+        # HTTP 2+ supports trailers on all requests/responses
+
+        flow.request.headers["trailer"] = "x-my-injected-trailer-header"
+        flow.request.trailers = Headers([
+            (b"x-my-injected-trailer-header", b"foobar")
+        ])
+        print("Injected a new request trailer...", flow.request.headers["trailer"])
+
 
 def response(flow: http.HTTPFlow):
     if flow.response.trailers:
         print("HTTP Trailers detected! Response contains:", flow.response.trailers)
 
     if flow.request.path == "/inject_trailers":
+        if flow.request.is_http10:
+            return
+        elif flow.request.is_http11:
+            if not flow.response.content:
+                return
+            flow.response.headers["transfer-encoding"] = "chunked"
+
         flow.response.headers["trailer"] = "x-my-injected-trailer-header"
         flow.response.trailers = Headers([
             (b"x-my-injected-trailer-header", b"foobar")
         ])
-        print("Injected a new trailer...", flow.response.headers["trailer"])
+        print("Injected a new response trailer...", flow.response.headers["trailer"])

mitmproxy/net/http/http1/assemble.py

@@ -5,7 +5,7 @@ def assemble_request(request):
     if request.data.content is None:
         raise exceptions.HttpException("Cannot assemble flow with missing content")
     head = assemble_request_head(request)
-    body = b"".join(assemble_body(request.data.headers, [request.data.content]))
+    body = b"".join(assemble_body(request.data.headers, [request.data.content], request.data.trailers))
     return head + body
 
 
@@ -19,7 +19,7 @@ def assemble_response(response):
     if response.data.content is None:
         raise exceptions.HttpException("Cannot assemble flow with missing content")
     head = assemble_response_head(response)
-    body = b"".join(assemble_body(response.data.headers, [response.data.content]))
+    body = b"".join(assemble_body(response.data.headers, [response.data.content], response.data.trailers))
     return head + body
 
 
@@ -29,13 +29,18 @@ def assemble_response_head(response):
     return b"%s\r\n%s\r\n" % (first_line, headers)
 
 
-def assemble_body(headers, body_chunks):
+def assemble_body(headers, body_chunks, trailers):
     if "chunked" in headers.get("transfer-encoding", "").lower():
         for chunk in body_chunks:
             if chunk:
                 yield b"%x\r\n%s\r\n" % (len(chunk), chunk)
+        if trailers:
+            yield b"0\r\n%s\r\n" % trailers
+        else:
             yield b"0\r\n\r\n"
     else:
+        if trailers:
+            raise exceptions.HttpException("Sending HTTP/1.1 trailer headers requires transfer-encoding: chunked")
         for chunk in body_chunks:
             yield chunk
 

mitmproxy/net/http/message.py

@@ -70,6 +70,14 @@ class Message(serializable.Serializable):
     def http_version(self, http_version: Union[str, bytes]) -> None:
         self.data.http_version = strutils.always_bytes(http_version, "utf-8", "surrogateescape")
 
+    @property
+    def is_http10(self) -> bool:
+        return self.data.http_version == b"HTTP/1.0"
+
+    @property
+    def is_http11(self) -> bool:
+        return self.data.http_version == b"HTTP/1.1"
+
     @property
     def is_http2(self) -> bool:
         return self.data.http_version == b"HTTP/2.0"

mitmproxy/proxy/protocol/http1.py

@@ -23,7 +23,7 @@ class Http1Layer(httpbase._HttpTransmissionLayer):
     def read_request_trailers(self, request):
         if "Trailer" in request.headers:
             # TODO: not implemented yet
-            self.log("HTTP/1 request trailer headers are not implemented yet!", "warn")
+            self.log("HTTP/1.1 request trailer headers are not implemented yet!", "warn")
         return None
 
     def send_request_headers(self, request):
@@ -32,17 +32,15 @@ class Http1Layer(httpbase._HttpTransmissionLayer):
         self.server_conn.wfile.flush()
 
     def send_request_body(self, request, chunks):
-        for chunk in http1.assemble_body(request.headers, chunks):
+        for chunk in http1.assemble_body(request.headers, chunks, request.trailers):
             self.server_conn.wfile.write(chunk)
             self.server_conn.wfile.flush()
 
     def send_request_trailers(self, request):
-        if "Trailer" in request.headers:
+        # HTTP/1.1 request trailer headers are sent in the body
-            # TODO: not implemented yet
+        pass
-            self.log("HTTP/1 request trailer headers are not implemented yet!", "warn")
 
     def send_request(self, request):
-        # TODO: this does not yet support request trailers
         self.server_conn.wfile.write(http1.assemble_request(request))
         self.server_conn.wfile.flush()
 
@@ -58,9 +56,10 @@ class Http1Layer(httpbase._HttpTransmissionLayer):
         )
 
     def read_response_trailers(self, request, response):
+        # Trailers should actually be parsed unconditionally, the "Trailer" header is optional
         if "Trailer" in response.headers:
             # TODO: not implemented yet
-            self.log("HTTP/1 trailer headers are not implemented yet!", "warn")
+            self.log("HTTP/1.1 trailer headers are not implemented yet!", "warn")
         return None
 
     def send_response_headers(self, response):
@@ -69,15 +68,13 @@ class Http1Layer(httpbase._HttpTransmissionLayer):
         self.client_conn.wfile.flush()
 
     def send_response_body(self, response, chunks):
-        for chunk in http1.assemble_body(response.headers, chunks):
+        for chunk in http1.assemble_body(response.headers, chunks, response.trailers):
             self.client_conn.wfile.write(chunk)
             self.client_conn.wfile.flush()
 
     def send_response_trailers(self, response):
-        if "Trailer" in response.headers:
+        # HTTP/1.1 response trailer headers are sent in the body
-            # TODO: not implemented yet
+        pass
-            self.log("HTTP/1 trailer headers are not implemented yet!", "warn")
-        return
 
     def check_close_connection(self, flow):
         request_close = http1.connection_close(

test/mitmproxy/net/http/http1/test_assemble.py

@@ -52,15 +52,21 @@ def test_assemble_response_head():
 
 
 def test_assemble_body():
-    c = list(assemble_body(Headers(), [b"body"]))
+    c = list(assemble_body(Headers(), [b"body"], Headers()))
     assert c == [b"body"]
 
-    c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a", b""]))
+    c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a", b""], Headers()))
     assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
 
-    c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"]))
+    c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"], Headers()))
     assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
 
+    c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"], Headers(trailer="trailer")))
+    assert c == [b"a\r\n123456789a\r\n", b"0\r\ntrailer: trailer\r\n\r\n"]
+
+    with pytest.raises(exceptions.HttpException):
+        list(assemble_body(Headers(), [b"body"], Headers(trailer="trailer")))
+
 
 def test_assemble_request_line():
     assert _assemble_request_line(treq().data) == b"GET /path HTTP/1.1"

test/mitmproxy/net/http/test_message.py

@@ -99,6 +99,9 @@ class TestMessage:
 
     def test_http_version(self):
         _test_decoded_attr(tutils.tresp(), "http_version")
+        assert tutils.tresp(http_version=b"HTTP/1.0").is_http10
+        assert tutils.tresp(http_version=b"HTTP/1.1").is_http11
+        assert tutils.tresp(http_version=b"HTTP/2.0").is_http2
 
 
 class TestMessageContentEncoding: