From 73ece6797c4b984dc913b9a78728fd7e390d53b2 Mon Sep 17 00:00:00 2001 From: sanlengjingvv Date: Thu, 25 Jun 2020 18:21:24 +0800 Subject: [PATCH 01/12] support HTTP2 trailers --- mitmproxy/proxy/protocol/http2.py | 42 +++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index f5e7b1c6d..dac9c41b2 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -55,7 +55,7 @@ class SafeH2Connection(connection.H2Connection): self.send_headers(stream_id, headers.fields, **kwargs) self.conn.send(self.data_to_send()) - def safe_send_body(self, raise_zombie: Callable, stream_id: int, chunks: List[bytes]): + def safe_send_body(self, raise_zombie: Callable, stream_id: int, chunks: List[bytes], end_stream=True): for chunk in chunks: position = 0 while position < len(chunk): @@ -75,10 +75,11 @@ class SafeH2Connection(connection.H2Connection): finally: self.lock.release() position += max_outbound_frame_size - with self.lock: - raise_zombie() - self.end_stream(stream_id) - self.conn.send(self.data_to_send()) + if end_stream: + with self.lock: + raise_zombie() + self.end_stream(stream_id) + self.conn.send(self.data_to_send()) class Http2Layer(base.Layer): @@ -170,7 +171,7 @@ class Http2Layer(base.Layer): elif isinstance(event, events.PriorityUpdated): return self._handle_priority_updated(eid, event) elif isinstance(event, events.TrailersReceived): - raise NotImplementedError('TrailersReceived not implemented') + return self._handle_trailers(eid, event, is_server, other_conn) # fail-safe for unhandled events return True @@ -233,6 +234,11 @@ class Http2Layer(base.Layer): self.connections[other_conn].safe_reset_stream(other_stream_id, event.error_code) return True + def _handle_trailers(self, eid, event, is_server, other_conn): + headers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers]) + self.streams[eid].update_trailers(headers) + return True + def _handle_remote_settings_changed(self, event, other_conn): new_settings = dict([(key, cs.new_value) for (key, cs) in event.changed_settings.items()]) self.connections[other_conn].safe_update_settings(new_settings) @@ -418,6 +424,8 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.response_data_finished = threading.Event() self.no_body = False + self.has_tailers = False + self.trailers_header = None self.priority_exclusive: bool self.priority_depends_on: Optional[int] = None @@ -591,6 +599,23 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr chunks ) + + @detect_zombie_stream + def update_trailers(self, headers): + self.trailers_header = headers + self.has_tailers = True + + @detect_zombie_stream + def send_trailers_headers(self): + if self.has_tailers and self.trailers_header: + with self.connections[self.client_conn].lock: + self.connections[self.client_conn].safe_send_headers( + self.raise_zombie, + self.client_stream_id, + self.trailers_header, + end_stream = True + ) + @detect_zombie_stream def send_request(self, message): self.send_request_headers(message) @@ -646,8 +671,11 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.connections[self.client_conn].safe_send_body( self.raise_zombie, self.client_stream_id, - chunks + chunks, + end_stream = not self.has_tailers ) + if self.has_tailers: + self.send_trailers_headers() def __call__(self): # pragma: no cover raise EnvironmentError('Http2SingleStreamLayer must be run as thread') From 7daecd24c5800163d8ac30b9fc7bc0a35f318802 Mon Sep 17 00:00:00 2001 From: sanlengjingvv Date: Fri, 26 Jun 2020 20:11:19 +0800 Subject: [PATCH 02/12] store trailers in HTTPFlow --- mitmproxy/http.py | 1 + mitmproxy/proxy/protocol/http.py | 3 +++ mitmproxy/proxy/protocol/http2.py | 5 ++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/mitmproxy/http.py b/mitmproxy/http.py index e99022243..0a063e657 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -140,6 +140,7 @@ class HTTPFlow(flow.Flow): """ request: HTTPRequest response: Optional[HTTPResponse] = None + trailers: http.Headers error: Optional[flow.Error] = None """ Note that it's possible for a Flow to have both a response and an error diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index 37e1669ee..7d734d8aa 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -409,6 +409,9 @@ class HttpLayer(base.Layer): self.log("response", "debug", [repr(f.response)]) self.channel.ask("response", f) + if hasattr(self, 'has_tailers') and self.has_tailers: + f.trailers = self.read_trailers_headers() + if not f.response.stream: # no streaming: # we already received the full response from the server and can diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index dac9c41b2..f4cee0483 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -599,7 +599,6 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr chunks ) - @detect_zombie_stream def update_trailers(self, headers): self.trailers_header = headers @@ -641,6 +640,10 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr timestamp_end=self.timestamp_end, ) + @detect_zombie_stream + def read_trailers_headers(self): + return self.trailers_header + @detect_zombie_stream def read_response_body(self, request, response): while True: From ca8285605ef75ce6794b4aa2e6be4f475fce2f32 Mon Sep 17 00:00:00 2001 From: sanlengjingvv Date: Sat, 27 Jun 2020 07:02:03 +0800 Subject: [PATCH 03/12] trailers should be Optional --- mitmproxy/http.py | 2 +- mitmproxy/proxy/protocol/http.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 0a063e657..78878a817 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -140,7 +140,7 @@ class HTTPFlow(flow.Flow): """ request: HTTPRequest response: Optional[HTTPResponse] = None - trailers: http.Headers + trailers: Optional[http.Headers] = None error: Optional[flow.Error] = None """ Note that it's possible for a Flow to have both a response and an error diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index 7d734d8aa..be05cd29f 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -407,10 +407,9 @@ class HttpLayer(base.Layer): self.channel.ask("responseheaders", f) self.log("response", "debug", [repr(f.response)]) - self.channel.ask("response", f) - if hasattr(self, 'has_tailers') and self.has_tailers: f.trailers = self.read_trailers_headers() + self.channel.ask("response", f) if not f.response.stream: # no streaming: From ebbba73281dceae4c486d62ab7b33dabd82cbfbb Mon Sep 17 00:00:00 2001 From: sanlengjingvv Date: Sat, 27 Jun 2020 15:41:14 +0800 Subject: [PATCH 04/12] serialize trailers for ws broadcast --- mitmproxy/tools/web/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 6bdd7eb1c..f6f975928 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -93,6 +93,8 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: "timestamp_end": flow.response.timestamp_end, "is_replay": flow.response.is_replay, } + if flow.trailers: + f["trailers"] = tuple(flow.trailers.items(True)) f.get("server_conn", {}).pop("cert", None) f.get("client_conn", {}).pop("mitmcert", None) From c87294dc60519ef48bb50996733e24591aff26df Mon Sep 17 00:00:00 2001 From: sanlengjingvv Date: Sun, 28 Jun 2020 00:51:44 +0800 Subject: [PATCH 05/12] move trailers to response and default None --- mitmproxy/http.py | 4 +++- mitmproxy/io/compat.py | 1 + mitmproxy/net/http/http1/read.py | 2 +- mitmproxy/net/http/response.py | 2 ++ mitmproxy/proxy/protocol/http.py | 2 +- mitmproxy/tools/web/app.py | 5 +++-- 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 78878a817..bf38a78e8 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -97,6 +97,7 @@ class HTTPResponse(http.Response): reason, headers, content, + trailers=None, timestamp_start=None, timestamp_end=None, is_replay=False @@ -108,6 +109,7 @@ class HTTPResponse(http.Response): reason, headers, content, + trailers, timestamp_start=timestamp_start, timestamp_end=timestamp_end, ) @@ -127,6 +129,7 @@ class HTTPResponse(http.Response): reason=response.data.reason, headers=response.data.headers, content=response.data.content, + trailers=response.data.trailers, timestamp_start=response.data.timestamp_start, timestamp_end=response.data.timestamp_end, ) @@ -140,7 +143,6 @@ class HTTPFlow(flow.Flow): """ request: HTTPRequest response: Optional[HTTPResponse] = None - trailers: Optional[http.Headers] = None error: Optional[flow.Error] = None """ Note that it's possible for a Flow to have both a response and an error diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index cddb8b01c..091a5db21 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -30,6 +30,7 @@ def convert_013_014(data): str(x) for x in data[b"response"].pop(b"httpversion")).encode() data[b"response"][b"status_code"] = data[b"response"].pop(b"code") data[b"response"][b"body"] = data[b"response"].pop(b"content") + data[b"response"][b"trailers"] = None data[b"server_conn"].pop(b"state") data[b"server_conn"][b"via"] = None data[b"version"] = (0, 14) diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py index a9585d7d4..0f60c8f4b 100644 --- a/mitmproxy/net/http/http1/read.py +++ b/mitmproxy/net/http/http1/read.py @@ -98,7 +98,7 @@ def read_response_head(rfile): # more accurate timestamp_start timestamp_start = rfile.first_byte_timestamp - return response.Response(http_version, status_code, message, headers, None, timestamp_start) + return response.Response(http_version, status_code, message, headers, None, None, timestamp_start) def read_body(rfile, expected_size, limit=None, max_chunk_size=4096): diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index c4dbf4080..edd8d4a62 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -22,6 +22,7 @@ class ResponseData(message.MessageData): reason=None, headers=(), content=None, + trailers=None, timestamp_start=None, timestamp_end=None ): @@ -39,6 +40,7 @@ class ResponseData(message.MessageData): self.reason = reason self.headers = headers self.content = content + self.trailers = trailers self.timestamp_start = timestamp_start self.timestamp_end = timestamp_end diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index be05cd29f..d4586079c 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -408,7 +408,7 @@ class HttpLayer(base.Layer): self.log("response", "debug", [repr(f.response)]) if hasattr(self, 'has_tailers') and self.has_tailers: - f.trailers = self.read_trailers_headers() + f.response.data.trailers = self.read_trailers_headers() self.channel.ask("response", f) if not f.response.stream: diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index f6f975928..2bbb60348 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -93,8 +93,9 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: "timestamp_end": flow.response.timestamp_end, "is_replay": flow.response.is_replay, } - if flow.trailers: - f["trailers"] = tuple(flow.trailers.items(True)) + if flow.response.data.trailers: + f["response"]["trailers"] = tuple(flow.response.data.trailers.items(True)) + f.get("server_conn", {}).pop("cert", None) f.get("client_conn", {}).pop("mitmcert", None) From 043d18ff731b6870129f6b8d4922436fb0d6d992 Mon Sep 17 00:00:00 2001 From: sanlengjingvv Date: Sun, 28 Jun 2020 01:13:20 +0800 Subject: [PATCH 06/12] correct spelling mistakes --- mitmproxy/proxy/protocol/http.py | 2 +- mitmproxy/proxy/protocol/http2.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index d4586079c..aa981b7a1 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -407,7 +407,7 @@ class HttpLayer(base.Layer): self.channel.ask("responseheaders", f) self.log("response", "debug", [repr(f.response)]) - if hasattr(self, 'has_tailers') and self.has_tailers: + if hasattr(self, 'has_trailers') and self.has_trailers: f.response.data.trailers = self.read_trailers_headers() self.channel.ask("response", f) diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index f4cee0483..f5ab09f09 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -424,7 +424,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.response_data_finished = threading.Event() self.no_body = False - self.has_tailers = False + self.has_trailers = False self.trailers_header = None self.priority_exclusive: bool @@ -602,11 +602,11 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr @detect_zombie_stream def update_trailers(self, headers): self.trailers_header = headers - self.has_tailers = True + self.has_trailers = True @detect_zombie_stream def send_trailers_headers(self): - if self.has_tailers and self.trailers_header: + if self.has_trailers and self.trailers_header: with self.connections[self.client_conn].lock: self.connections[self.client_conn].safe_send_headers( self.raise_zombie, @@ -675,9 +675,9 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.raise_zombie, self.client_stream_id, chunks, - end_stream = not self.has_tailers + end_stream = not self.has_trailers ) - if self.has_tailers: + if self.has_trailers: self.send_trailers_headers() def __call__(self): # pragma: no cover From 22eb492a13ccbd2dca7fbe51d49433314990b8c2 Mon Sep 17 00:00:00 2001 From: sanlengjingvv Date: Thu, 2 Jul 2020 18:32:03 +0800 Subject: [PATCH 07/12] update FLOW_FORMAT_VERSION version fix error when dump flow with http trailers add testcase for http trailers --- mitmproxy/io/compat.py | 6 +- mitmproxy/net/http/message.py | 4 + mitmproxy/proxy/protocol/http.py | 3 +- mitmproxy/version.py | 2 +- test/mitmproxy/proxy/protocol/test_http2.py | 85 +++++++++++++++++++++ 5 files changed, 97 insertions(+), 3 deletions(-) diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index 091a5db21..0d5e5513f 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -30,7 +30,6 @@ def convert_013_014(data): str(x) for x in data[b"response"].pop(b"httpversion")).encode() data[b"response"][b"status_code"] = data[b"response"].pop(b"code") data[b"response"][b"body"] = data[b"response"].pop(b"content") - data[b"response"][b"trailers"] = None data[b"server_conn"].pop(b"state") data[b"server_conn"][b"via"] = None data[b"version"] = (0, 14) @@ -172,6 +171,10 @@ def convert_6_7(data): data["client_conn"]["tls_extensions"] = None return data +def convert_7_8(data): + data["version"] = 8 + data["response"]["trailers"] = None + return data def _convert_dict_keys(o: Any) -> Any: if isinstance(o, dict): @@ -227,6 +230,7 @@ converters = { 4: convert_4_5, 5: convert_5_6, 6: convert_6_7, + 7: convert_7_8, } diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index af7b032b3..478d334e4 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -28,6 +28,8 @@ class MessageData(serializable.Serializable): def get_state(self): state = vars(self).copy() state["headers"] = state["headers"].get_state() + if 'trailers' in state and state["trailers"] is not None: + state["trailers"] = state["trailers"].get_state() return state @classmethod @@ -53,6 +55,8 @@ class Message(serializable.Serializable): @classmethod def from_state(cls, state): state["headers"] = mheaders.Headers.from_state(state["headers"]) + if 'trailers' in state and state["trailers"] is not None: + state["trailers"] = mheaders.Headers.from_state(state["trailers"]) return cls(**state) @property diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index aa981b7a1..7949e7e21 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -407,7 +407,8 @@ class HttpLayer(base.Layer): self.channel.ask("responseheaders", f) self.log("response", "debug", [repr(f.response)]) - if hasattr(self, 'has_trailers') and self.has_trailers: + # not support HTTP/1.1 trailers + if f.request.http_version == "HTTP/2.0": f.response.data.trailers = self.read_trailers_headers() self.channel.ask("response", f) diff --git a/mitmproxy/version.py b/mitmproxy/version.py index c55ccdf76..f31ec7116 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -8,7 +8,7 @@ MITMPROXY = "mitmproxy " + VERSION # Serialization format version. This is displayed nowhere, it just needs to be incremented by one # for each change in the file format. -FLOW_FORMAT_VERSION = 7 +FLOW_FORMAT_VERSION = 8 def get_dev_version() -> str: diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 0afa63050..fd338c6c3 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -1031,3 +1031,88 @@ class TestResponseStreaming(_Http2Test): assert data else: assert data is None + + +class TestTrailers(_Http2Test): + request_body_buffer = b'' + + @classmethod + def handle_server_event(cls, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.RequestReceived): + assert (b'self.client-foo', b'self.client-bar-1') in event.headers + assert (b'self.client-foo', b'self.client-bar-2') in event.headers + elif isinstance(event, h2.events.StreamEnded): + import warnings + with warnings.catch_warnings(): + # Ignore UnicodeWarning: + # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison + # failed to convert both arguments to Unicode - interpreting + # them as being unequal. + # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: + + warnings.simplefilter("ignore") + h2_conn.send_headers(event.stream_id, [ + (':status', '200'), + ('server-foo', 'server-bar'), + ('föo', 'bär'), + ('X-Stream-ID', str(event.stream_id)), + ]) + h2_conn.send_data(event.stream_id, b'response body') + h2_conn.send_headers(event.stream_id, [ + ('trailers', 'trailers-foo'), + ], end_stream=True) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + elif isinstance(event, h2.events.DataReceived): + cls.request_body_buffer += event.data + return True + + def test_trailers(self): + response_body_buffer = b'' + h2_conn = self.setup_connection() + + self._send_request( + self.client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ('self.client-FoO', 'self.client-bar-1'), + ('self.client-FoO', 'self.client-bar-2'), + ], + body=b'request body') + + done = False + while not done: + try: + raw = b''.join(http2.read_raw_frame(self.client.rfile)) + events = h2_conn.receive_data(raw) + except exceptions.HttpException: + print(traceback.format_exc()) + assert False + + self.client.wfile.write(h2_conn.data_to_send()) + self.client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.DataReceived): + response_body_buffer += event.data + elif isinstance(event, h2.events.StreamEnded): + done = True + + h2_conn.close_connection() + self.client.wfile.write(h2_conn.data_to_send()) + self.client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert self.master.state.flows[0].response.status_code == 200 + assert self.master.state.flows[0].response.headers['server-foo'] == 'server-bar' + assert self.master.state.flows[0].response.headers['föo'] == 'bär' + assert self.master.state.flows[0].response.content == b'response body' + assert self.request_body_buffer == b'request body' + assert response_body_buffer == b'response body' + assert self.master.state.flows[0].response.data.trailers['trailers'] == 'trailers-foo' From d589f13a1d34bad85cae94d4001c7aa624f79af0 Mon Sep 17 00:00:00 2001 From: sanlengjingvv Date: Thu, 2 Jul 2020 18:46:38 +0800 Subject: [PATCH 08/12] fix lint error --- mitmproxy/io/compat.py | 2 ++ test/mitmproxy/proxy/protocol/test_http2.py | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index 0d5e5513f..e9d74e1c6 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -171,11 +171,13 @@ def convert_6_7(data): data["client_conn"]["tls_extensions"] = None return data + def convert_7_8(data): data["version"] = 8 data["response"]["trailers"] = None return data + def _convert_dict_keys(o: Any) -> Any: if isinstance(o, dict): return {strutils.always_str(k): _convert_dict_keys(v) for k, v in o.items()} diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index fd338c6c3..d6870de45 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -1060,9 +1060,7 @@ class TestTrailers(_Http2Test): ('X-Stream-ID', str(event.stream_id)), ]) h2_conn.send_data(event.stream_id, b'response body') - h2_conn.send_headers(event.stream_id, [ - ('trailers', 'trailers-foo'), - ], end_stream=True) + h2_conn.send_headers(event.stream_id, [('trailers', 'trailers-foo')], end_stream=True) wfile.write(h2_conn.data_to_send()) wfile.flush() elif isinstance(event, h2.events.DataReceived): From ebb061796cde538905a8f250962cc67baf9785fc Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 2 Jul 2020 16:05:02 +0200 Subject: [PATCH 09/12] unify HTTP trailers APIs --- mitmproxy/http.py | 3 + mitmproxy/io/compat.py | 1 + mitmproxy/net/http/http1/read.py | 2 +- mitmproxy/net/http/message.py | 14 ++++ mitmproxy/net/http/request.py | 4 ++ mitmproxy/net/http/response.py | 2 + mitmproxy/proxy/protocol/http.py | 22 +++++- mitmproxy/proxy/protocol/http1.py | 24 +++++++ mitmproxy/proxy/protocol/http2.py | 78 +++++++++++++-------- pathod/protocols/http2.py | 5 +- test/mitmproxy/net/http/test_request.py | 3 + test/mitmproxy/net/http/test_response.py | 3 + test/mitmproxy/proxy/protocol/test_http2.py | 42 +++-------- 13 files changed, 137 insertions(+), 66 deletions(-) diff --git a/mitmproxy/http.py b/mitmproxy/http.py index bf38a78e8..2cd5bccc9 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -26,6 +26,7 @@ class HTTPRequest(http.Request): http_version, headers, content, + trailers=None, timestamp_start=None, timestamp_end=None, is_replay=False, @@ -41,6 +42,7 @@ class HTTPRequest(http.Request): http_version, headers, content, + trailers, timestamp_start, timestamp_end, ) @@ -73,6 +75,7 @@ class HTTPRequest(http.Request): http_version=request.data.http_version, headers=request.data.headers, content=request.data.content, + trailers=request.data.trailers, timestamp_start=request.data.timestamp_start, timestamp_end=request.data.timestamp_end, ) diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index e9d74e1c6..16e157756 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -174,6 +174,7 @@ def convert_6_7(data): def convert_7_8(data): data["version"] = 8 + data["request"]["trailers"] = None data["response"]["trailers"] = None return data diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py index 0f60c8f4b..ce2007ed9 100644 --- a/mitmproxy/net/http/http1/read.py +++ b/mitmproxy/net/http/http1/read.py @@ -59,7 +59,7 @@ def read_request_head(rfile): timestamp_start = rfile.first_byte_timestamp return request.Request( - form, method, scheme, host, port, path, http_version, headers, None, timestamp_start + form, method, scheme, host, port, path, http_version, headers, None, None, timestamp_start ) diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index 478d334e4..4a16f52aa 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -134,6 +134,20 @@ class Message(serializable.Serializable): content = property(get_content, set_content) + @property + def trailers(self): + """ + Message trailers object + + Returns: + mitmproxy.net.http.Headers + """ + return self.data.trailers + + @trailers.setter + def trailers(self, h): + self.data.trailers = h + @property def http_version(self): """ diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index ba699e2aa..243378cf4 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -29,6 +29,7 @@ class RequestData(message.MessageData): http_version, headers=(), content=None, + trailers=None, timestamp_start=None, timestamp_end=None ): @@ -46,6 +47,8 @@ class RequestData(message.MessageData): headers = nheaders.Headers(headers) if isinstance(content, str): raise ValueError("Content must be bytes, not {}".format(type(content).__name__)) + if trailers is not None and not isinstance(trailers, nheaders.Headers): + trailers = nheaders.Headers(trailers) self.first_line_format = first_line_format self.method = method @@ -56,6 +59,7 @@ class RequestData(message.MessageData): self.http_version = http_version self.headers = headers self.content = content + self.trailers = trailers self.timestamp_start = timestamp_start self.timestamp_end = timestamp_end diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index edd8d4a62..7cc41940f 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -34,6 +34,8 @@ class ResponseData(message.MessageData): headers = nheaders.Headers(headers) if isinstance(content, str): raise ValueError("Content must be bytes, not {}".format(type(content).__name__)) + if trailers is not None and not isinstance(trailers, nheaders.Headers): + trailers = nheaders.Headers(trailers) self.http_version = http_version self.status_code = status_code diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index 7949e7e21..c2f3779df 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -20,6 +20,9 @@ class _HttpTransmissionLayer(base.Layer): def read_request_body(self, request): raise NotImplementedError() + def read_request_trailers(self, request): + raise NotImplementedError() + def send_request(self, request): raise NotImplementedError() @@ -30,11 +33,15 @@ class _HttpTransmissionLayer(base.Layer): raise NotImplementedError() yield "this is a generator" # pragma: no cover + def read_response_trailers(self, request, response): + raise NotImplementedError() + def read_response(self, request): response = self.read_response_headers() response.data.content = b"".join( self.read_response_body(request, response) ) + response.data.trailers = self.read_response_trailers(request, response) return response def send_response(self, response): @@ -42,6 +49,7 @@ class _HttpTransmissionLayer(base.Layer): raise exceptions.HttpException("Cannot assemble flow with missing content") self.send_response_headers(response) self.send_response_body(response, [response.data.content]) + self.send_response_trailers(response) def send_response_headers(self, response): raise NotImplementedError() @@ -49,6 +57,9 @@ class _HttpTransmissionLayer(base.Layer): def send_response_body(self, response, chunks): raise NotImplementedError() + def send_response_trailers(self, response, chunks): + raise NotImplementedError() + def check_close_connection(self, f): raise NotImplementedError() @@ -255,6 +266,7 @@ class HttpLayer(base.Layer): f.request.data.content = b"".join( self.read_request_body(f.request) ) + f.request.data.trailers = self.read_request_trailers(f.request) f.request.timestamp_end = time.time() self.channel.ask("http_connect", f) @@ -282,6 +294,9 @@ class HttpLayer(base.Layer): f.request.data.content = None else: f.request.data.content = b"".join(self.read_request_body(request)) + + f.request.data.trailers = self.read_request_trailers(f.request) + request.timestamp_end = time.time() except exceptions.HttpException as e: # We optimistically guess there might be an HTTP client on the @@ -348,6 +363,8 @@ class HttpLayer(base.Layer): else: self.send_request_body(f.request, [f.request.data.content]) + self.send_request_trailers(f.request) + f.response = self.read_response_headers() try: @@ -406,10 +423,9 @@ class HttpLayer(base.Layer): # we now need to emulate the responseheaders hook. self.channel.ask("responseheaders", f) + f.response.data.trailers = self.read_response_trailers(f.request, f.response) + self.log("response", "debug", [repr(f.response)]) - # not support HTTP/1.1 trailers - if f.request.http_version == "HTTP/2.0": - f.response.data.trailers = self.read_trailers_headers() self.channel.ask("response", f) if not f.response.stream: diff --git a/mitmproxy/proxy/protocol/http1.py b/mitmproxy/proxy/protocol/http1.py index 91f1e9b7d..5fc4efbaf 100644 --- a/mitmproxy/proxy/protocol/http1.py +++ b/mitmproxy/proxy/protocol/http1.py @@ -23,6 +23,12 @@ class Http1Layer(httpbase._HttpTransmissionLayer): human.parse_size(self.config.options.body_size_limit) ) + def read_request_trailers(self, request): + if "Trailer" in request.headers: + # TODO: not implemented yet + self.log("HTTP/1 request trailer headers are not implemented yet!", "warn") + return None + def send_request_headers(self, request): headers = http1.assemble_request_head(request) self.server_conn.wfile.write(headers) @@ -33,7 +39,13 @@ class Http1Layer(httpbase._HttpTransmissionLayer): self.server_conn.wfile.write(chunk) self.server_conn.wfile.flush() + def send_request_trailers(self, request): + if "Trailer" in request.headers: + # TODO: not implemented yet + self.log("HTTP/1 request trailer headers are not implemented yet!", "warn") + def send_request(self, request): + # TODO: this does not yet support request trailers self.server_conn.wfile.write(http1.assemble_request(request)) self.server_conn.wfile.flush() @@ -49,6 +61,12 @@ class Http1Layer(httpbase._HttpTransmissionLayer): human.parse_size(self.config.options.body_size_limit) ) + def read_response_trailers(self, request, response): + if "Trailer" in response.headers: + # TODO: not implemented yet + self.log("HTTP/1 trailer headers are not implemented yet!", "warn") + return None + def send_response_headers(self, response): raw = http1.assemble_response_head(response) self.client_conn.wfile.write(raw) @@ -59,6 +77,12 @@ class Http1Layer(httpbase._HttpTransmissionLayer): self.client_conn.wfile.write(chunk) self.client_conn.wfile.flush() + def send_response_trailers(self, response): + if "Trailer" in response.headers: + # TODO: not implemented yet + self.log("HTTP/1 trailer headers are not implemented yet!", "warn") + return + def check_close_connection(self, flow): request_close = http1.connection_close( flow.request.http_version, diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index f5ab09f09..9275e6bdb 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -235,8 +235,10 @@ class Http2Layer(base.Layer): return True def _handle_trailers(self, eid, event, is_server, other_conn): - headers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers]) - self.streams[eid].update_trailers(headers) + trailers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers]) + # TODO: support request trailers as well! + self.streams[eid].response_trailers = trailers + self.streams[eid].response_trailers_arrived.set() return True def _handle_remote_settings_changed(self, event, other_conn): @@ -417,15 +419,17 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.request_data_queue: queue.Queue[bytes] = queue.Queue() self.request_queued_data_length = 0 self.request_data_finished = threading.Event() + self.request_trailers_arrived = threading.Event() + self.request_trailers = None self.response_arrived = threading.Event() self.response_data_queue: queue.Queue[bytes] = queue.Queue() self.response_queued_data_length = 0 self.response_data_finished = threading.Event() + self.response_trailers_arrived = threading.Event() + self.response_trailers = None self.no_body = False - self.has_trailers = False - self.trailers_header = None self.priority_exclusive: bool self.priority_depends_on: Optional[int] = None @@ -437,8 +441,10 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.zombie = time.time() self.request_data_finished.set() self.request_arrived.set() + self.request_trailers_arrived.set() self.response_arrived.set() self.response_data_finished.set() + self.response_trailers_arrived.set() def connect(self): # pragma: no cover raise exceptions.Http2ProtocolException("HTTP2 layer should already have a connection.") @@ -526,6 +532,14 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr break self.raise_zombie() + @detect_zombie_stream + def read_request_trailers(self, request): + if "trailer" in request.headers: + self.request_trailers_arrived.wait() + self.raise_zombie() + return self.request_trailers + return None + @detect_zombie_stream def send_request_headers(self, request): if self.pushed: @@ -600,25 +614,14 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr ) @detect_zombie_stream - def update_trailers(self, headers): - self.trailers_header = headers - self.has_trailers = True + def send_request_trailers(self, request): + self._send_trailers(self.server_conn, self.request_trailers) @detect_zombie_stream - def send_trailers_headers(self): - if self.has_trailers and self.trailers_header: - with self.connections[self.client_conn].lock: - self.connections[self.client_conn].safe_send_headers( - self.raise_zombie, - self.client_stream_id, - self.trailers_header, - end_stream = True - ) - - @detect_zombie_stream - def send_request(self, message): - self.send_request_headers(message) - self.send_request_body(message, [message.content]) + def send_request(self, request): + self.send_request_headers(request) + self.send_request_body(request, [request.content]) + self.send_request_trailers(request) @detect_zombie_stream def read_response_headers(self): @@ -640,10 +643,6 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr timestamp_end=self.timestamp_end, ) - @detect_zombie_stream - def read_trailers_headers(self): - return self.trailers_header - @detect_zombie_stream def read_response_body(self, request, response): while True: @@ -658,6 +657,14 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr break self.raise_zombie() + @detect_zombie_stream + def read_response_trailers(self, request, response): + if "trailer" in response.headers: + self.response_trailers_arrived.wait() + self.raise_zombie() + return self.response_trailers + return None + @detect_zombie_stream def send_response_headers(self, response): headers = response.headers.copy() @@ -670,15 +677,28 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr ) @detect_zombie_stream - def send_response_body(self, _response, chunks): + def send_response_body(self, response, chunks): self.connections[self.client_conn].safe_send_body( self.raise_zombie, self.client_stream_id, chunks, - end_stream = not self.has_trailers + end_stream=("trailer" not in response.headers) ) - if self.has_trailers: - self.send_trailers_headers() + + @detect_zombie_stream + def send_response_trailers(self, _response): + self._send_trailers(self.client_conn, self.response_trailers) + + def _send_trailers(self, conn, trailers): + if not trailers: + return + with self.connections[conn].lock: + self.connections[conn].safe_send_headers( + self.raise_zombie, + self.client_stream_id, + trailers, + end_stream=True + ) def __call__(self): # pragma: no cover raise EnvironmentError('Http2SingleStreamLayer must be run as thread') diff --git a/pathod/protocols/http2.py b/pathod/protocols/http2.py index c56d304db..748893ee2 100644 --- a/pathod/protocols/http2.py +++ b/pathod/protocols/http2.py @@ -110,8 +110,9 @@ class HTTP2StateProtocol: b"HTTP/2.0", headers, body, - timestamp_start, - timestamp_end, + None, + timestamp_start=timestamp_start, + timestamp_end=timestamp_end, ) request.stream_id = stream_id diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py index 71d5c7a12..30129d331 100644 --- a/test/mitmproxy/net/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -21,8 +21,11 @@ class TestRequestData: treq(headers="foobar") with pytest.raises(ValueError): treq(content="foobar") + with pytest.raises(ValueError): + treq(trailers="foobar") assert isinstance(treq(headers=()).headers, Headers) + assert isinstance(treq(trailers=()).trailers, Headers) class TestRequestCore: diff --git a/test/mitmproxy/net/http/test_response.py b/test/mitmproxy/net/http/test_response.py index 08d72840e..7eb3eab82 100644 --- a/test/mitmproxy/net/http/test_response.py +++ b/test/mitmproxy/net/http/test_response.py @@ -20,8 +20,11 @@ class TestResponseData: tresp(reason="fööbär") with pytest.raises(ValueError): tresp(content="foobar") + with pytest.raises(ValueError): + tresp(trailers="foobar") assert isinstance(tresp(headers=()).headers, Headers) + assert isinstance(tresp(trailers=()).trailers, Headers) class TestResponseCore: diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index d6870de45..1529e7317 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -1034,37 +1034,19 @@ class TestResponseStreaming(_Http2Test): class TestTrailers(_Http2Test): - request_body_buffer = b'' - @classmethod def handle_server_event(cls, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): return False - elif isinstance(event, h2.events.RequestReceived): - assert (b'self.client-foo', b'self.client-bar-1') in event.headers - assert (b'self.client-foo', b'self.client-bar-2') in event.headers elif isinstance(event, h2.events.StreamEnded): - import warnings - with warnings.catch_warnings(): - # Ignore UnicodeWarning: - # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison - # failed to convert both arguments to Unicode - interpreting - # them as being unequal. - # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: - - warnings.simplefilter("ignore") - h2_conn.send_headers(event.stream_id, [ - (':status', '200'), - ('server-foo', 'server-bar'), - ('föo', 'bär'), - ('X-Stream-ID', str(event.stream_id)), - ]) + h2_conn.send_headers(event.stream_id, [ + (':status', '200'), + ('trailer', 'x-my-trailers') + ]) h2_conn.send_data(event.stream_id, b'response body') - h2_conn.send_headers(event.stream_id, [('trailers', 'trailers-foo')], end_stream=True) + h2_conn.send_headers(event.stream_id, [('x-my-trailers', 'foobar')], end_stream=True) wfile.write(h2_conn.data_to_send()) wfile.flush() - elif isinstance(event, h2.events.DataReceived): - cls.request_body_buffer += event.data return True def test_trailers(self): @@ -1079,11 +1061,9 @@ class TestTrailers(_Http2Test): (':method', 'GET'), (':scheme', 'https'), (':path', '/'), - ('self.client-FoO', 'self.client-bar-1'), - ('self.client-FoO', 'self.client-bar-2'), - ], - body=b'request body') + ]) + trailers_buffer = None done = False while not done: try: @@ -1099,6 +1079,8 @@ class TestTrailers(_Http2Test): for event in events: if isinstance(event, h2.events.DataReceived): response_body_buffer += event.data + elif isinstance(event, h2.events.TrailersReceived): + trailers_buffer = event.headers elif isinstance(event, h2.events.StreamEnded): done = True @@ -1108,9 +1090,7 @@ class TestTrailers(_Http2Test): assert len(self.master.state.flows) == 1 assert self.master.state.flows[0].response.status_code == 200 - assert self.master.state.flows[0].response.headers['server-foo'] == 'server-bar' - assert self.master.state.flows[0].response.headers['föo'] == 'bär' assert self.master.state.flows[0].response.content == b'response body' - assert self.request_body_buffer == b'request body' assert response_body_buffer == b'response body' - assert self.master.state.flows[0].response.data.trailers['trailers'] == 'trailers-foo' + assert self.master.state.flows[0].response.data.trailers['x-my-trailers'] == 'foobar' + assert trailers_buffer == [(b'x-my-trailers', b'foobar')] From 288ce65d73551a13c818733c78689206110e9ef1 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 3 Jul 2020 12:49:31 +0200 Subject: [PATCH 10/12] add h2 trailers example and fix sending --- examples/addons/http-trailers.py | 26 ++++++++++++++++++++++++++ mitmproxy/proxy/protocol/http2.py | 6 +++--- 2 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 examples/addons/http-trailers.py diff --git a/examples/addons/http-trailers.py b/examples/addons/http-trailers.py new file mode 100644 index 000000000..77b9d1a40 --- /dev/null +++ b/examples/addons/http-trailers.py @@ -0,0 +1,26 @@ +""" +This script simply prints all received HTTP Trailers. + +HTTP requests and responses can container trailing headers which are sent after +the body is fully transmitted. Such trailers need to be announced in the initial +headers by name, so the receiving endpoint can wait and read them after the +body. +""" + +from mitmproxy import http +from mitmproxy.net.http import Headers + +def request(flow: http.HTTPFlow): + if flow.request.trailers: + print("HTTP Trailers detected! Request contains:", flow.request.trailers) + +def response(flow: http.HTTPFlow): + if flow.response.trailers: + print("HTTP Trailers detected! Response contains:", flow.response.trailers) + + if flow.request.path == "/inject_trailers": + flow.response.headers["trailer"] = "x-my-injected-trailer-header" + flow.response.trailers = Headers([ + (b"x-my-injected-trailer-header", b"foobar") + ]) + print("Injected a new trailer...", flow.response.headers["trailer"]) diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index 9275e6bdb..602946f65 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -615,7 +615,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr @detect_zombie_stream def send_request_trailers(self, request): - self._send_trailers(self.server_conn, self.request_trailers) + self._send_trailers(self.server_conn, request.trailers) @detect_zombie_stream def send_request(self, request): @@ -686,8 +686,8 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr ) @detect_zombie_stream - def send_response_trailers(self, _response): - self._send_trailers(self.client_conn, self.response_trailers) + def send_response_trailers(self, response): + self._send_trailers(self.client_conn, response.trailers) def _send_trailers(self, conn, trailers): if not trailers: From 828ba0c2e79d8c54806a1c9eefb6007abc8aabc0 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 6 Jul 2020 01:01:48 +0200 Subject: [PATCH 11/12] refactor h2 trailer support This allow's trailers without the initial Trailer header announcement. In HTTP/2 the stream ends with any frame containing END_SREAM. In the case of trailers, it is a final HEADERS frame after all the DATA frames. Therefore we do not need to explicitly check for the trailer announcement header, but can simply wait until the response message / stream has ended. --- examples/addons/http-trailers.py | 2 + mitmproxy/proxy/protocol/http2.py | 141 +++++++++++++++--------------- 2 files changed, 73 insertions(+), 70 deletions(-) diff --git a/examples/addons/http-trailers.py b/examples/addons/http-trailers.py index 77b9d1a40..d85965c13 100644 --- a/examples/addons/http-trailers.py +++ b/examples/addons/http-trailers.py @@ -10,10 +10,12 @@ body. from mitmproxy import http from mitmproxy.net.http import Headers + def request(flow: http.HTTPFlow): if flow.request.trailers: print("HTTP Trailers detected! Request contains:", flow.request.trailers) + def response(flow: http.HTTPFlow): if flow.response.trailers: print("HTTP Trailers detected! Response contains:", flow.response.trailers) diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index 602946f65..5da91ac24 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -180,22 +180,22 @@ class Http2Layer(base.Layer): headers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers]) self.streams[eid] = Http2SingleStreamLayer(self, self.connections[self.client_conn], eid, headers) self.streams[eid].timestamp_start = time.time() - self.streams[eid].no_body = (event.stream_ended is not None) + self.streams[eid].no_request_body = (event.stream_ended is not None) if event.priority_updated is not None: self.streams[eid].priority_exclusive = event.priority_updated.exclusive self.streams[eid].priority_depends_on = event.priority_updated.depends_on self.streams[eid].priority_weight = event.priority_updated.weight self.streams[eid].handled_priority_event = event.priority_updated self.streams[eid].start() - self.streams[eid].request_arrived.set() + self.streams[eid].request_message.arrived.set() return True def _handle_response_received(self, eid, event): headers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers]) self.streams[eid].queued_data_length = 0 self.streams[eid].timestamp_start = time.time() - self.streams[eid].response_headers = headers - self.streams[eid].response_arrived.set() + self.streams[eid].response_message.headers = headers + self.streams[eid].response_message.arrived.set() return True def _handle_data_received(self, eid, event, source_conn): @@ -220,7 +220,7 @@ class Http2Layer(base.Layer): def _handle_stream_ended(self, eid): self.streams[eid].timestamp_end = time.time() - self.streams[eid].data_finished.set() + self.streams[eid].stream_ended.set() return True def _handle_stream_reset(self, eid, event, is_server, other_conn): @@ -236,9 +236,7 @@ class Http2Layer(base.Layer): def _handle_trailers(self, eid, event, is_server, other_conn): trailers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers]) - # TODO: support request trailers as well! - self.streams[eid].response_trailers = trailers - self.streams[eid].response_trailers_arrived.set() + self.streams[eid].trailers = trailers return True def _handle_remote_settings_changed(self, event, other_conn): @@ -285,8 +283,8 @@ class Http2Layer(base.Layer): self.streams[event.pushed_stream_id].pushed = True self.streams[event.pushed_stream_id].parent_stream_id = parent_eid self.streams[event.pushed_stream_id].timestamp_end = time.time() - self.streams[event.pushed_stream_id].request_arrived.set() - self.streams[event.pushed_stream_id].request_data_finished.set() + self.streams[event.pushed_stream_id].request_message.arrived.set() + self.streams[event.pushed_stream_id].request_message.stream_ended.set() self.streams[event.pushed_stream_id].start() return True @@ -400,6 +398,16 @@ def detect_zombie_stream(func): # pragma: no cover class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThread): + class Message: + def __init__(self, headers=None): + self.headers: Optional[mitmproxy.net.http.Headers] = headers # headers are the first thing to be received on a new stream + self.data_queue: queue.Queue[bytes] = queue.Queue() # contains raw contents of DATA frames + self.queued_data_length = 0 # used to enforce mitmproxy's config.options.body_size_limit + self.trailers: Optional[mitmproxy.net.http.Headers] = None # trailers are received after stream_ended is set + + self.arrived = threading.Event() # indicates the HEADERS+CONTINUTATION frames have been received + self.stream_ended = threading.Event() # indicates the a frame with the END_STREAM flag has been received + def __init__(self, ctx, h2_connection, stream_id: int, request_headers: mitmproxy.net.http.Headers) -> None: super().__init__( ctx, name="Http2SingleStreamLayer-{}".format(stream_id) @@ -408,28 +416,15 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.zombie: Optional[float] = None self.client_stream_id: int = stream_id self.server_stream_id: Optional[int] = None - self.request_headers = request_headers - self.response_headers: Optional[mitmproxy.net.http.Headers] = None self.pushed = False self.timestamp_start: Optional[float] = None self.timestamp_end: Optional[float] = None - self.request_arrived = threading.Event() - self.request_data_queue: queue.Queue[bytes] = queue.Queue() - self.request_queued_data_length = 0 - self.request_data_finished = threading.Event() - self.request_trailers_arrived = threading.Event() - self.request_trailers = None + self.request_message = self.Message(request_headers) + self.response_message = self.Message() - self.response_arrived = threading.Event() - self.response_data_queue: queue.Queue[bytes] = queue.Queue() - self.response_queued_data_length = 0 - self.response_data_finished = threading.Event() - self.response_trailers_arrived = threading.Event() - self.response_trailers = None - - self.no_body = False + self.no_request_body = False self.priority_exclusive: bool self.priority_depends_on: Optional[int] = None @@ -439,12 +434,10 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr def kill(self): if not self.zombie: self.zombie = time.time() - self.request_data_finished.set() - self.request_arrived.set() - self.request_trailers_arrived.set() - self.response_arrived.set() - self.response_data_finished.set() - self.response_trailers_arrived.set() + self.request_message.stream_ended.set() + self.request_message.arrived.set() + self.response_message.arrived.set() + self.response_message.stream_ended.set() def connect(self): # pragma: no cover raise exceptions.Http2ProtocolException("HTTP2 layer should already have a connection.") @@ -462,28 +455,44 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr @property def data_queue(self): - if self.response_arrived.is_set(): - return self.response_data_queue + if self.response_message.arrived.is_set(): + return self.response_message.data_queue else: - return self.request_data_queue + return self.request_message.data_queue @property def queued_data_length(self): - if self.response_arrived.is_set(): - return self.response_queued_data_length + if self.response_message.arrived.is_set(): + return self.response_message.queued_data_length else: - return self.request_queued_data_length + return self.request_message.queued_data_length @queued_data_length.setter def queued_data_length(self, v): - self.request_queued_data_length = v + self.request_message.queued_data_length = v @property - def data_finished(self): - if self.response_arrived.is_set(): - return self.response_data_finished + def stream_ended(self): + # This indicates that all message headers, the full message body, and all trailers have been received + # https://tools.ietf.org/html/rfc7540#section-8.1 + if self.response_message.arrived.is_set(): + return self.response_message.stream_ended else: - return self.request_data_finished + return self.request_message.stream_ended + + @property + def trailers(self): + if self.response_message.arrived.is_set(): + return self.response_message.trailers + else: + return self.request_message.trailers + + @trailers.setter + def trailers(self, v): + if self.response_message.arrived.is_set(): + self.response_message.trailers = v + else: + self.request_message.trailers = v def raise_zombie(self, pre_command=None): # pragma: no cover connection_closed = self.h2_connection.state_machine.state == h2.connection.ConnectionState.CLOSED @@ -494,13 +503,13 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr @detect_zombie_stream def read_request_headers(self, flow): - self.request_arrived.wait() + self.request_message.arrived.wait() self.raise_zombie() if self.pushed: flow.metadata['h2-pushed-stream'] = True - first_line_format, method, scheme, host, port, path = http2.parse_headers(self.request_headers) + first_line_format, method, scheme, host, port, path = http2.parse_headers(self.request_message.headers) return http.HTTPRequest( first_line_format, method, @@ -509,7 +518,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr port, path, b"HTTP/2.0", - self.request_headers, + self.request_message.headers, None, timestamp_start=self.timestamp_start, timestamp_end=self.timestamp_end, @@ -518,27 +527,23 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr @detect_zombie_stream def read_request_body(self, request): if not request.stream: - self.request_data_finished.wait() + self.request_message.stream_ended.wait() while True: try: - yield self.request_data_queue.get(timeout=0.1) + yield self.request_message.data_queue.get(timeout=0.1) except queue.Empty: # pragma: no cover pass - if self.request_data_finished.is_set(): + if self.request_message.stream_ended.is_set(): self.raise_zombie() - while self.request_data_queue.qsize() > 0: - yield self.request_data_queue.get() + while self.request_message.data_queue.qsize() > 0: + yield self.request_message.data_queue.get() break self.raise_zombie() @detect_zombie_stream def read_request_trailers(self, request): - if "trailer" in request.headers: - self.request_trailers_arrived.wait() - self.raise_zombie() - return self.request_trailers - return None + return self.request_message.trailers @detect_zombie_stream def send_request_headers(self, request): @@ -589,7 +594,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.raise_zombie, self.server_stream_id, headers, - end_stream=self.no_body, + end_stream=self.no_request_body, priority_exclusive=priority_exclusive, priority_depends_on=priority_depends_on, priority_weight=priority_weight, @@ -606,7 +611,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr # nothing to do here return - if not self.no_body: + if not self.no_request_body: self.connections[self.server_conn].safe_send_body( self.raise_zombie, self.server_stream_id, @@ -625,12 +630,12 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr @detect_zombie_stream def read_response_headers(self): - self.response_arrived.wait() + self.response_message.arrived.wait() self.raise_zombie() - status_code = int(self.response_headers.get(':status', 502)) - headers = self.response_headers.copy() + status_code = int(self.response_message.headers.get(':status', 502)) + headers = self.response_message.headers.copy() headers.pop(":status", None) return http.HTTPResponse( @@ -647,23 +652,19 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr def read_response_body(self, request, response): while True: try: - yield self.response_data_queue.get(timeout=0.1) + yield self.response_message.data_queue.get(timeout=0.1) except queue.Empty: # pragma: no cover pass - if self.response_data_finished.is_set(): + if self.response_message.stream_ended.is_set(): self.raise_zombie() - while self.response_data_queue.qsize() > 0: - yield self.response_data_queue.get() + while self.response_message.data_queue.qsize() > 0: + yield self.response_message.data_queue.get() break self.raise_zombie() @detect_zombie_stream def read_response_trailers(self, request, response): - if "trailer" in response.headers: - self.response_trailers_arrived.wait() - self.raise_zombie() - return self.response_trailers - return None + return self.response_message.trailers @detect_zombie_stream def send_response_headers(self, response): From c0f62cc559ae4f0778d6079a2b31d2ce0309dda6 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 6 Jul 2020 12:36:54 +0200 Subject: [PATCH 12/12] fix missing message body and end_stream for trailers --- mitmproxy/proxy/protocol/http2.py | 18 ++-- test/mitmproxy/proxy/protocol/test_http2.py | 93 +++++++++++++++++++-- 2 files changed, 94 insertions(+), 17 deletions(-) diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index 5da91ac24..c8aaed8ab 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -180,7 +180,6 @@ class Http2Layer(base.Layer): headers = mitmproxy.net.http.Headers([[k, v] for k, v in event.headers]) self.streams[eid] = Http2SingleStreamLayer(self, self.connections[self.client_conn], eid, headers) self.streams[eid].timestamp_start = time.time() - self.streams[eid].no_request_body = (event.stream_ended is not None) if event.priority_updated is not None: self.streams[eid].priority_exclusive = event.priority_updated.exclusive self.streams[eid].priority_depends_on = event.priority_updated.depends_on @@ -424,8 +423,6 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.request_message = self.Message(request_headers) self.response_message = self.Message() - self.no_request_body = False - self.priority_exclusive: bool self.priority_depends_on: Optional[int] = None self.priority_weight: Optional[int] = None @@ -594,7 +591,6 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.raise_zombie, self.server_stream_id, headers, - end_stream=self.no_request_body, priority_exclusive=priority_exclusive, priority_depends_on=priority_depends_on, priority_weight=priority_weight, @@ -611,12 +607,12 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr # nothing to do here return - if not self.no_request_body: - self.connections[self.server_conn].safe_send_body( - self.raise_zombie, - self.server_stream_id, - chunks - ) + self.connections[self.server_conn].safe_send_body( + self.raise_zombie, + self.server_stream_id, + chunks, + end_stream=(request.trailers is None), + ) @detect_zombie_stream def send_request_trailers(self, request): @@ -683,7 +679,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.raise_zombie, self.client_stream_id, chunks, - end_stream=("trailer" not in response.headers) + end_stream=(response.trailers is None), ) @detect_zombie_stream diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 1529e7317..ba1070102 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -1033,29 +1033,110 @@ class TestResponseStreaming(_Http2Test): assert data is None -class TestTrailers(_Http2Test): +class TestRequestTrailers(_Http2Test): + server_trailers_received = False + + @classmethod + def handle_server_event(cls, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.RequestReceived): + # reset the value for a fresh test + cls.server_trailers_received = False + elif isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.TrailersReceived): + cls.server_trailers_received = True + + elif isinstance(event, h2.events.StreamEnded): + h2_conn.send_headers(event.stream_id, [ + (':status', '200'), + ('x-my-trailer-request-received', 'success' if cls.server_trailers_received else "failure"), + ], end_stream=True) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + @pytest.mark.parametrize('announce', [True, False]) + @pytest.mark.parametrize('body', [None, b"foobar"]) + def test_trailers(self, announce, body): + h2_conn = self.setup_connection() + stream_id = 1 + headers = [ + (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ] + if announce: + headers.append(('trailer', 'x-my-trailers')) + h2_conn.send_headers( + stream_id=stream_id, + headers=headers, + ) + if body: + h2_conn.send_data(stream_id, body) + + # send trailers + h2_conn.send_headers(stream_id, [('x-my-trailers', 'foobar')], end_stream=True) + + self.client.wfile.write(h2_conn.data_to_send()) + self.client.wfile.flush() + + done = False + while not done: + try: + raw = b''.join(http2.read_raw_frame(self.client.rfile)) + events = h2_conn.receive_data(raw) + except exceptions.HttpException: + print(traceback.format_exc()) + assert False + + self.client.wfile.write(h2_conn.data_to_send()) + self.client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + done = True + + h2_conn.close_connection() + self.client.wfile.write(h2_conn.data_to_send()) + self.client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert self.master.state.flows[0].request.trailers['x-my-trailers'] == 'foobar' + assert self.master.state.flows[0].response.status_code == 200 + assert self.master.state.flows[0].response.headers['x-my-trailer-request-received'] == 'success' + + +class TestResponseTrailers(_Http2Test): + @classmethod def handle_server_event(cls, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): return False elif isinstance(event, h2.events.StreamEnded): - h2_conn.send_headers(event.stream_id, [ + headers = [ (':status', '200'), - ('trailer', 'x-my-trailers') - ]) + ] + if event.stream_id == 1: + # special stream_id to activate the Trailer announcement header + headers.append(('trailer', 'x-my-trailers')) + + h2_conn.send_headers(event.stream_id, headers) h2_conn.send_data(event.stream_id, b'response body') h2_conn.send_headers(event.stream_id, [('x-my-trailers', 'foobar')], end_stream=True) wfile.write(h2_conn.data_to_send()) wfile.flush() return True - def test_trailers(self): + @pytest.mark.parametrize('announce', [True, False]) + def test_trailers(self, announce): response_body_buffer = b'' h2_conn = self.setup_connection() self._send_request( self.client.wfile, h2_conn, + stream_id=(1 if announce else 3), headers=[ (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), (':method', 'GET'), @@ -1092,5 +1173,5 @@ class TestTrailers(_Http2Test): assert self.master.state.flows[0].response.status_code == 200 assert self.master.state.flows[0].response.content == b'response body' assert response_body_buffer == b'response body' - assert self.master.state.flows[0].response.data.trailers['x-my-trailers'] == 'foobar' + assert self.master.state.flows[0].response.trailers['x-my-trailers'] == 'foobar' assert trailers_buffer == [(b'x-my-trailers', b'foobar')]