From b9e3b46fd87729685ff7e2326b2d31f744578e28 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 5 Apr 2020 00:30:28 +0200 Subject: [PATCH] [sans-io] add http fuzzing tests --- mitmproxy/proxy2/context.py | 1 - mitmproxy/proxy2/layers/http/__init__.py | 3 +- mitmproxy/proxy2/layers/http/_http1.py | 37 ++++-- .../mitmproxy/proxy2/layers/test_http_fuzz.py | 125 ++++++++++++++++++ 4 files changed, 151 insertions(+), 15 deletions(-) create mode 100644 test/mitmproxy/proxy2/layers/test_http_fuzz.py diff --git a/mitmproxy/proxy2/context.py b/mitmproxy/proxy2/context.py index ca58692b8..e69c30e0d 100644 --- a/mitmproxy/proxy2/context.py +++ b/mitmproxy/proxy2/context.py @@ -58,7 +58,6 @@ class Client(Connection): class Server(Connection): state = ConnectionState.CLOSED - peername = None sockname = None address: Optional[tuple] diff --git a/mitmproxy/proxy2/layers/http/__init__.py b/mitmproxy/proxy2/layers/http/__init__.py index 3ac8e7155..b9537754c 100644 --- a/mitmproxy/proxy2/layers/http/__init__.py +++ b/mitmproxy/proxy2/layers/http/__init__.py @@ -368,7 +368,8 @@ class HttpStream(layer.Layer): yield from self.child_layer.handle_event(events.Start()) self._handle_event = self.passthrough else: - return (yield from self.send_response()) + yield from self.send_response() + return SendHttp(ResponseProtocolError(self.stream_id, "EOF"), self.context.client) @expect(RequestData, RequestEndOfMessage, events.Event) def passthrough(self, event: events.Event) -> layer.CommandGenerator[None]: diff --git a/mitmproxy/proxy2/layers/http/_http1.py b/mitmproxy/proxy2/layers/http/_http1.py index 041e64c91..a2fdd5cae 100644 --- a/mitmproxy/proxy2/layers/http/_http1.py +++ b/mitmproxy/proxy2/layers/http/_http1.py @@ -5,7 +5,7 @@ import h11 from h11._readers import ChunkedReader, ContentLengthReader, Http10Reader from h11._receivebuffer import ReceiveBuffer -from mitmproxy import http +from mitmproxy import exceptions, http from mitmproxy.net.http import http1 from mitmproxy.net.http.http1 import read_sansio as http1_sansio from mitmproxy.proxy2 import commands, events, layer @@ -167,12 +167,14 @@ class Http1Server(Http1Connection): if request_head: request_head = [bytes(x) for x in request_head] # TODO: Make url.parse compatible with bytearrays try: - self.request = http.HTTPRequest.wrap(http1_sansio.read_request_head(request_head)) - except ValueError as e: - yield commands.Log(f"{human.format_address(self.conn.address)}: {e}") + req = http1_sansio.read_request_head(request_head) + expected_body_size = http1.expected_http_body_size(req, expect_continue_as_0=False) + except (ValueError, exceptions.HttpSyntaxException) as e: + yield commands.Log(f"{human.format_address(self.conn.peername)}: {e}") yield commands.CloseConnection(self.conn) self.state = self.wait return + self.request = http.HTTPRequest.wrap(req) yield ReceiveHttp(RequestHeaders(self.stream_id, self.request)) if self.request.first_line_format == "authority": @@ -182,8 +184,7 @@ class Http1Server(Http1Connection): # https://http2.github.io/http2-spec/#CONNECT self.state = self.wait else: - expected_size = http1.expected_http_body_size(self.request, expect_continue_as_0=False) - self.body_reader = self.make_body_reader(expected_size) + self.body_reader = self.make_body_reader(expected_body_size) self.state = self.read_request_body yield from self.state(event) else: @@ -271,14 +272,24 @@ class Http1Client(Http1Connection): @expect(events.ConnectionEvent) def read_response_headers(self, event: events.ConnectionEvent) -> layer.CommandGenerator[None]: if isinstance(event, events.DataReceived): + if not self.request: + # we just received some data for an unknown request. + yield commands.Log(f"Unexpected data from server: {bytes(self.buf)!r}") + yield commands.CloseConnection(self.conn) + return + response_head = self.buf.maybe_extract_lines() - if response_head: - response_head = [bytes(x) for x in response_head] - self.response = http.HTTPResponse.wrap(http1_sansio.read_response_head(response_head)) + response_head = [bytes(x) for x in response_head] # TODO: Make url.parse compatible with bytearrays + try: + resp = http1_sansio.read_response_head(response_head) + expected_size = http1.expected_http_body_size(self.request, resp) + except (ValueError, exceptions.HttpSyntaxException) as e: + yield ReceiveHttp(ResponseProtocolError(self.stream_id, f"Cannot parse HTTP response: {e}")) + yield commands.CloseConnection(self.conn) + return + self.response = http.HTTPResponse.wrap(resp) yield ReceiveHttp(ResponseHeaders(self.stream_id, self.response)) - - expected_size = http1.expected_http_body_size(self.request, self.response) self.body_reader = self.make_body_reader(expected_size) self.state = self.read_response_body @@ -288,8 +299,8 @@ class Http1Client(Http1Connection): elif isinstance(event, events.ConnectionClosed): if self.stream_id: if self.buf: - yield ReceiveHttp( - ResponseProtocolError(self.stream_id, f"unexpected server response: {bytes(self.buf)}")) + yield ReceiveHttp(ResponseProtocolError(self.stream_id, + f"unexpected server response: {bytes(self.buf)!r}")) else: # The server has closed the connection to prevent us from continuing. # We need to signal that to the stream. diff --git a/test/mitmproxy/proxy2/layers/test_http_fuzz.py b/test/mitmproxy/proxy2/layers/test_http_fuzz.py new file mode 100644 index 000000000..b454db5f7 --- /dev/null +++ b/test/mitmproxy/proxy2/layers/test_http_fuzz.py @@ -0,0 +1,125 @@ +import os +from typing import Iterable + +import pytest +from hypothesis import example, given, seed, settings +from hypothesis.strategies import binary, composite, integers, lists, sampled_from + +from mitmproxy import options +from mitmproxy.addons.proxyserver import Proxyserver +from mitmproxy.proxy.protocol.http import HTTPMode +from mitmproxy.proxy2 import context, events +from mitmproxy.proxy2.commands import OpenConnection, SendData +from mitmproxy.proxy2.events import DataReceived, Start +from mitmproxy.proxy2.layers import http +from test.mitmproxy.proxy2.tutils import Placeholder, Playbook, reply + +settings.register_profile("full", max_examples=100_000, deadline=None) +settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) + + +@pytest.fixture(scope="module") +def opts(): + opts = options.Options() + Proxyserver().load(opts) + return opts + + +request_lines = sampled_from([ + b"GET / HTTP/1.1", + b"GET http://example.com/ HTTP/1.1", + b"CONNECT example.com:443 HTTP/1.1", + b"HEAD /foo HTTP/0.9", +]) +response_lines = sampled_from([ + b"HTTP/1.1 200 OK", + b"HTTP/1.1 100 Continue", + b"HTTP/0.9 204 No Content", + b"HEAD /foo HTTP/0.9", +]) +headers = lists(sampled_from([ + b"Host: example.com", + b"Content-Length: 5", + b"Expect: 100-continue", + b"Transfer-Encoding: chunked", + b"Connection: close", + b"", +])) +bodies = sampled_from([ + b"", + b"12345", + b"5\r\n12345\r\n0\r\n\r\n" +]) + + +def mutate(draw, data: bytes) -> bytes: + cut_start = draw(integers(0, len(data))) + cut_end = draw(integers(cut_start, len(data))) + data = data[:cut_start] + data[cut_end:] + + replace_start = draw(integers(0, len(data))) + replace_end = draw(integers(replace_start, len(data))) + return data[:replace_start] + draw(binary()) + data[replace_end:] + + +def split(draw, data: bytes) -> Iterable[bytes]: + a, b = sorted([ + draw(integers(0, len(data))), + draw(integers(0, len(data))) + ]) + if a > 0: + yield data[:a] + if a != b: + yield data[a:b] + if b < len(data): + yield data[b:] + + +@composite +def fuzz_request(draw): + request = draw(request_lines) + b"\r\n" + request += b"\r\n".join(draw(headers)) + request += b"\r\n\r\n" + draw(bodies) + request = mutate(draw, request) + request = list(split(draw, request)) + return request + + +@composite +def fuzz_response(draw): + response = draw(response_lines) + b"\r\n" + response += b"\r\n".join(draw(headers)) + response += b"\r\n\r\n" + draw(bodies) + response = mutate(draw, response) + response = list(split(draw, response)) + return response + + +@given(fuzz_request()) +def test_fuzz_request(opts, data): + tctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080)), opts) + + layer = http.HttpLayer(tctx, HTTPMode.regular) + for _ in layer.handle_event(Start()): + pass + for chunk in data: + for _ in layer.handle_event(DataReceived(tctx.client, chunk)): + pass + + +@given(fuzz_response()) +@example([b'0 OK\r\n\r\n', b'\r\n', b'5\r\n12345\r\n0\r\n\r\n']) +def test_fuzz_response(opts, data): + tctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080)), opts) + server = Placeholder() + playbook = Playbook(http.HttpLayer(tctx, HTTPMode.regular), hooks=False) + assert ( + playbook + >> DataReceived(tctx.client, b"GET http://example.com/ HTTP/1.1\r\nHost: example.com\r\n\r\n") + << OpenConnection(server) + >> reply(None) + << SendData(server, b"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n") + ) + for chunk in data: + for _ in playbook.layer.handle_event(events.DataReceived(server(), chunk)): + pass