diff --git a/CHANGELOG b/CHANGELOG index 980dc26cc..13754c226 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -18,6 +18,7 @@ Unreleased: mitmproxy next * Add interactive tutorials to the documentation (@mplattner) * Support `deflateRaw` `Content-Encoding`s (@kjoconnor) * Fix broken requests without body on HTTP/2 (@Kriechi) + * Add support for sending (but not parsing) HTTP Trailers to the HTTP/1.1 protocol (@bburky) * --- TODO: add new PRs above this line --- diff --git a/docs/src/content/concepts-protocols.md b/docs/src/content/concepts-protocols.md index 73e0aae47..9a3dd6e48 100644 --- a/docs/src/content/concepts-protocols.md +++ b/docs/src/content/concepts-protocols.md @@ -16,7 +16,7 @@ menu: HTTP/1.0 and HTTP/1.1 support in mitmproxy is based on our custom HTTP stack, which takes care of all semantics and on-the-wire parsing/serialization tasks. -mitmproxy currently does not support HTTP trailers - but if you want to send +mitmproxy currently does not support parsing HTTP trailers - but if you want to send us a PR, we promise to take look! ## HTTP/2 diff --git a/examples/addons/http-trailers.py b/examples/addons/http-trailers.py index d85965c13..ba0732ba1 100644 --- a/examples/addons/http-trailers.py +++ b/examples/addons/http-trailers.py @@ -15,14 +15,40 @@ def request(flow: http.HTTPFlow): if flow.request.trailers: print("HTTP Trailers detected! Request contains:", flow.request.trailers) + if flow.request.path == "/inject_trailers": + if flow.request.is_http10: + # HTTP/1.0 doesn't support trailers + return + elif flow.request.is_http11: + if not flow.request.content: + # Avoid sending a body on GET requests or a 0 byte chunked body with trailers. + # Otherwise some servers return 400 Bad Request. + return + # HTTP 1.1 requires transfer-encoding: chunked to send trailers + flow.request.headers["transfer-encoding"] = "chunked" + # HTTP 2+ supports trailers on all requests/responses + + flow.request.headers["trailer"] = "x-my-injected-trailer-header" + flow.request.trailers = Headers([ + (b"x-my-injected-trailer-header", b"foobar") + ]) + print("Injected a new request trailer...", flow.request.headers["trailer"]) + def response(flow: http.HTTPFlow): if flow.response.trailers: print("HTTP Trailers detected! Response contains:", flow.response.trailers) if flow.request.path == "/inject_trailers": + if flow.request.is_http10: + return + elif flow.request.is_http11: + if not flow.response.content: + return + flow.response.headers["transfer-encoding"] = "chunked" + flow.response.headers["trailer"] = "x-my-injected-trailer-header" flow.response.trailers = Headers([ (b"x-my-injected-trailer-header", b"foobar") ]) - print("Injected a new trailer...", flow.response.headers["trailer"]) + print("Injected a new response trailer...", flow.response.headers["trailer"]) diff --git a/mitmproxy/net/http/http1/assemble.py b/mitmproxy/net/http/http1/assemble.py index 1437f09fc..ac72fab89 100644 --- a/mitmproxy/net/http/http1/assemble.py +++ b/mitmproxy/net/http/http1/assemble.py @@ -5,7 +5,7 @@ def assemble_request(request): if request.data.content is None: raise exceptions.HttpException("Cannot assemble flow with missing content") head = assemble_request_head(request) - body = b"".join(assemble_body(request.data.headers, [request.data.content])) + body = b"".join(assemble_body(request.data.headers, [request.data.content], request.data.trailers)) return head + body @@ -19,7 +19,7 @@ def assemble_response(response): if response.data.content is None: raise exceptions.HttpException("Cannot assemble flow with missing content") head = assemble_response_head(response) - body = b"".join(assemble_body(response.data.headers, [response.data.content])) + body = b"".join(assemble_body(response.data.headers, [response.data.content], response.data.trailers)) return head + body @@ -29,13 +29,18 @@ def assemble_response_head(response): return b"%s\r\n%s\r\n" % (first_line, headers) -def assemble_body(headers, body_chunks): +def assemble_body(headers, body_chunks, trailers): if "chunked" in headers.get("transfer-encoding", "").lower(): for chunk in body_chunks: if chunk: yield b"%x\r\n%s\r\n" % (len(chunk), chunk) - yield b"0\r\n\r\n" + if trailers: + yield b"0\r\n%s\r\n" % trailers + else: + yield b"0\r\n\r\n" else: + if trailers: + raise exceptions.HttpException("Sending HTTP/1.1 trailer headers requires transfer-encoding: chunked") for chunk in body_chunks: yield chunk diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index ba3269aa0..3a1d0577b 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -70,6 +70,14 @@ class Message(serializable.Serializable): def http_version(self, http_version: Union[str, bytes]) -> None: self.data.http_version = strutils.always_bytes(http_version, "utf-8", "surrogateescape") + @property + def is_http10(self) -> bool: + return self.data.http_version == b"HTTP/1.0" + + @property + def is_http11(self) -> bool: + return self.data.http_version == b"HTTP/1.1" + @property def is_http2(self) -> bool: return self.data.http_version == b"HTTP/2.0" diff --git a/mitmproxy/proxy/protocol/http1.py b/mitmproxy/proxy/protocol/http1.py index 48198e789..dbd567f2d 100644 --- a/mitmproxy/proxy/protocol/http1.py +++ b/mitmproxy/proxy/protocol/http1.py @@ -23,7 +23,7 @@ class Http1Layer(httpbase._HttpTransmissionLayer): def read_request_trailers(self, request): if "Trailer" in request.headers: # TODO: not implemented yet - self.log("HTTP/1 request trailer headers are not implemented yet!", "warn") + self.log("HTTP/1.1 request trailer headers are not implemented yet!", "warn") return None def send_request_headers(self, request): @@ -32,17 +32,15 @@ class Http1Layer(httpbase._HttpTransmissionLayer): self.server_conn.wfile.flush() def send_request_body(self, request, chunks): - for chunk in http1.assemble_body(request.headers, chunks): + for chunk in http1.assemble_body(request.headers, chunks, request.trailers): self.server_conn.wfile.write(chunk) self.server_conn.wfile.flush() def send_request_trailers(self, request): - if "Trailer" in request.headers: - # TODO: not implemented yet - self.log("HTTP/1 request trailer headers are not implemented yet!", "warn") + # HTTP/1.1 request trailer headers are sent in the body + pass def send_request(self, request): - # TODO: this does not yet support request trailers self.server_conn.wfile.write(http1.assemble_request(request)) self.server_conn.wfile.flush() @@ -58,9 +56,10 @@ class Http1Layer(httpbase._HttpTransmissionLayer): ) def read_response_trailers(self, request, response): + # Trailers should actually be parsed unconditionally, the "Trailer" header is optional if "Trailer" in response.headers: # TODO: not implemented yet - self.log("HTTP/1 trailer headers are not implemented yet!", "warn") + self.log("HTTP/1.1 trailer headers are not implemented yet!", "warn") return None def send_response_headers(self, response): @@ -69,15 +68,13 @@ class Http1Layer(httpbase._HttpTransmissionLayer): self.client_conn.wfile.flush() def send_response_body(self, response, chunks): - for chunk in http1.assemble_body(response.headers, chunks): + for chunk in http1.assemble_body(response.headers, chunks, response.trailers): self.client_conn.wfile.write(chunk) self.client_conn.wfile.flush() def send_response_trailers(self, response): - if "Trailer" in response.headers: - # TODO: not implemented yet - self.log("HTTP/1 trailer headers are not implemented yet!", "warn") - return + # HTTP/1.1 response trailer headers are sent in the body + pass def check_close_connection(self, flow): request_close = http1.connection_close( diff --git a/test/mitmproxy/net/http/http1/test_assemble.py b/test/mitmproxy/net/http/http1/test_assemble.py index 3b1b073c8..4fea192fc 100644 --- a/test/mitmproxy/net/http/http1/test_assemble.py +++ b/test/mitmproxy/net/http/http1/test_assemble.py @@ -52,15 +52,21 @@ def test_assemble_response_head(): def test_assemble_body(): - c = list(assemble_body(Headers(), [b"body"])) + c = list(assemble_body(Headers(), [b"body"], Headers())) assert c == [b"body"] - c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a", b""])) + c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a", b""], Headers())) assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"] - c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"])) + c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"], Headers())) assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"] + c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"], Headers(trailer="trailer"))) + assert c == [b"a\r\n123456789a\r\n", b"0\r\ntrailer: trailer\r\n\r\n"] + + with pytest.raises(exceptions.HttpException): + list(assemble_body(Headers(), [b"body"], Headers(trailer="trailer"))) + def test_assemble_request_line(): assert _assemble_request_line(treq().data) == b"GET /path HTTP/1.1" diff --git a/test/mitmproxy/net/http/test_message.py b/test/mitmproxy/net/http/test_message.py index bd42c30c5..16c7ac30c 100644 --- a/test/mitmproxy/net/http/test_message.py +++ b/test/mitmproxy/net/http/test_message.py @@ -99,6 +99,9 @@ class TestMessage: def test_http_version(self): _test_decoded_attr(tutils.tresp(), "http_version") + assert tutils.tresp(http_version=b"HTTP/1.0").is_http10 + assert tutils.tresp(http_version=b"HTTP/1.1").is_http11 + assert tutils.tresp(http_version=b"HTTP/2.0").is_http2 class TestMessageContentEncoding: