[sans-io] split http layer into smaller modules

mitmproxy/proxy2/layers/__init__.py

@@ -1,5 +1,5 @@
 from . import modes
-from .http.http import HTTPLayer
+from .http import HTTPLayer
 from .tcp import TCPLayer
 from .tls import ClientTLSLayer, ServerTLSLayer
 

mitmproxy/proxy2/layers/http/__init__.py

@@ -0,0 +1,5 @@
+from .http import HTTPLayer
+
+__all__ = [
+    "HTTPLayer",
+]

mitmproxy/proxy2/layers/http/base.py

@@ -0,0 +1,37 @@
+import abc
+import typing
+
+from mitmproxy.proxy2 import commands, events
+
+StreamId = int
+
+
+class HttpEvent(events.Event):
+    stream_id: StreamId
+
+    # we need stream ids on every event to avoid race conditions
+
+    def __init__(self, stream_id: StreamId):
+        self.stream_id = stream_id
+
+    def __repr__(self) -> str:
+        x = self.__dict__.copy()
+        x.pop("stream_id")
+        return f"{type(self).__name__}({repr(x) if x else ''})"
+
+
+class HttpConnection(abc.ABC):
+    @abc.abstractmethod
+    def handle_event(self, event: events.Event) -> typing.Iterator[HttpEvent]:
+        yield from ()
+
+    @abc.abstractmethod
+    def send(self, event: HttpEvent) -> commands.TCommandGenerator:
+        yield from ()
+
+
+__all__ = [
+    "HttpConnection",
+    "StreamId",
+    "HttpEvent",
+]

mitmproxy/proxy2/layers/http/events.py

@@ -0,0 +1,53 @@
+from mitmproxy import http
+from .base import HttpEvent, StreamId
+
+
+class RequestHeaders(HttpEvent):
+    request: http.HTTPRequest
+
+    def __init__(self, request: http.HTTPRequest, stream_id: StreamId):
+        super().__init__(stream_id)
+        self.request = request
+
+
+class ResponseHeaders(HttpEvent):
+    response: http.HTTPResponse
+
+    def __init__(self, response: http.HTTPResponse, stream_id: StreamId):
+        super().__init__(stream_id)
+        self.response = response
+
+
+class RequestData(HttpEvent):
+    data: bytes
+
+    def __init__(self, data: bytes, stream_id: StreamId):
+        super().__init__(stream_id)
+        self.data = data
+
+
+class ResponseData(HttpEvent):
+    data: bytes
+
+    def __init__(self, data: bytes, stream_id: StreamId):
+        super().__init__(stream_id)
+        self.data = data
+
+
+class RequestEndOfMessage(HttpEvent):
+    pass
+
+
+class ResponseEndOfMessage(HttpEvent):
+    pass
+
+
+__all__ = [
+    "HttpEvent",
+    "RequestHeaders",
+    "RequestData",
+    "RequestEndOfMessage",
+    "ResponseHeaders",
+    "ResponseData",
+    "ResponseEndOfMessage",
+]

mitmproxy/proxy2/layers/http/http.py

@@ -1,37 +1,19 @@
 import collections
 import typing
-from abc import ABC, abstractmethod
-
-import h11
-from h11._readers import ChunkedReader, ContentLengthReader, Http10Reader
-from h11._receivebuffer import ReceiveBuffer
 
 from mitmproxy import flow, http
-from mitmproxy.net.http import http1
-from mitmproxy.net.http.http1 import read_sansio as http1_sansio
 from mitmproxy.proxy.protocol.http import HTTPMode
 from mitmproxy.proxy2 import commands, events
-from mitmproxy.proxy2.context import Client, Connection, Context, Server
+from mitmproxy.proxy2.context import Connection, Context, Server
 from mitmproxy.proxy2.layer import Layer, NextLayer
 from mitmproxy.proxy2.layers.tls import EstablishServerTLS, EstablishServerTLSReply, HTTP_ALPNS
 from mitmproxy.proxy2.utils import expect
 from mitmproxy.utils import human
-
+from .base import HttpConnection, StreamId
-StreamId = int
+from .events import HttpEvent, RequestData, RequestEndOfMessage, RequestHeaders, ResponseData, ResponseEndOfMessage, \
-
+    ResponseHeaders
-
+from .http1 import Http1Client, Http1Server
-class HttpEvent(events.Event):
+from .http2 import Http2Client
-    stream_id: StreamId
-
-    # we need stream ids on every event to avoid race conditions
-
-    def __init__(self, stream_id: StreamId):
-        self.stream_id = stream_id
-
-    def __repr__(self) -> str:
-        x = self.__dict__.copy()
-        x.pop("stream_id")
-        return f"{type(self).__name__}({repr(x) if x else ''})"
 
 
 class HttpCommand(commands.Command):
@@ -76,287 +58,6 @@ class SendHttp(HttpCommand):
         return f"Send({self.event})"
 
 
-HttpEventGenerator = typing.Iterator[HttpEvent]
-
-
-class RequestHeaders(HttpEvent):
-    request: http.HTTPRequest
-
-    def __init__(self, request: http.HTTPRequest, stream_id: StreamId):
-        super().__init__(stream_id)
-        self.request = request
-
-
-class ResponseHeaders(HttpEvent):
-    response: http.HTTPResponse
-
-    def __init__(self, response: http.HTTPResponse, stream_id: StreamId):
-        super().__init__(stream_id)
-        self.response = response
-
-
-class RequestData(HttpEvent):
-    data: bytes
-
-    def __init__(self, data: bytes, stream_id: StreamId):
-        super().__init__(stream_id)
-        self.data = data
-
-
-class ResponseData(HttpEvent):
-    data: bytes
-
-    def __init__(self, data: bytes, stream_id: StreamId):
-        super().__init__(stream_id)
-        self.data = data
-
-
-class RequestEndOfMessage(HttpEvent):
-    pass
-
-
-class ResponseEndOfMessage(HttpEvent):
-    pass
-
-
-TBodyReader = typing.Union[ChunkedReader, Http10Reader, ContentLengthReader]
-
-
-class Http1Connection(ABC):
-    conn: Connection
-    stream_id: StreamId = None
-    request: http.HTTPRequest
-    response: http.HTTPResponse
-    state: typing.Callable[[events.Event], HttpEventGenerator]
-    body_reader: TBodyReader
-    buf: ReceiveBuffer
-
-    def __init__(self, conn: Connection):
-        assert isinstance(conn, Connection)
-        self.conn = conn
-        self.buf = ReceiveBuffer()
-
-    def handle_event(self, event: events.Event) -> HttpEventGenerator:
-        if isinstance(event, events.DataReceived):
-            self.buf += event.data
-        yield from self.state(event)
-
-    @abstractmethod
-    def send(self, event: HttpEvent) -> commands.TCommandGenerator:
-        yield from ()
-
-    def make_body_reader(self, expected_size: typing.Optional[int]) -> TBodyReader:
-        if expected_size is None:
-            return ChunkedReader()
-        elif expected_size == -1:
-            return Http10Reader()
-        else:
-            return ContentLengthReader(expected_size)
-
-    def read_body(self, event: events.Event, is_request: bool) -> typing.Iterator[HttpEvent]:
-        while True:
-            try:
-                if isinstance(event, events.DataReceived):
-                    h11_event = self.body_reader(self.buf)
-                elif isinstance(event, events.ConnectionClosed):
-                    h11_event = self.body_reader.read_eof()
-                else:
-                    raise ValueError(f"Unexpected event: {event}")
-            except h11.ProtocolError:
-                raise  # FIXME
-
-            if h11_event is None:
-                return
-            elif isinstance(h11_event, h11.Data):
-                h11_event.data: bytearray  # type checking
-                if is_request:
-                    yield RequestData(bytes(h11_event.data), self.stream_id)
-                else:
-                    yield ResponseData(bytes(h11_event.data), self.stream_id)
-            elif isinstance(h11_event, h11.EndOfMessage):
-                if is_request:
-                    yield RequestEndOfMessage(self.stream_id)
-                else:
-                    yield ResponseEndOfMessage(self.stream_id)
-                return
-
-    def wait(self, event: events.Event) -> HttpEventGenerator:
-        """
-        We wait for the current flow to be finished before parsing the next message,
-        as we may want to upgrade to WebSocket or plain TCP before that.
-        """
-        if isinstance(event, events.DataReceived):
-            return
-        elif isinstance(event, events.ConnectionClosed):
-            return
-        else:
-            yield from ()
-            raise ValueError(f"Unexpected event: {event}")
-
-
-class Http1Server(Http1Connection):
-    """A simple HTTP/1 server with no pipelining support."""
-    conn: Client
-
-    def __init__(self, conn: Client):
-        super().__init__(conn)
-        self.state = self.read_request_headers
-        self.stream_id = -1
-
-    def send(self, event: HttpEvent) -> commands.TCommandGenerator:
-        if isinstance(event, ResponseHeaders):
-            self.response = event.response
-            raw = http1.assemble_response_head(event.response)
-            if self.request.first_line_format == "authority":
-                assert self.state == self.wait
-                self.body_reader = self.make_body_reader(-1)
-                self.state = self.read_request_body
-                yield from self.state(events.DataReceived(self.conn, b""))
-        elif isinstance(event, ResponseData):
-            if "chunked" in self.response.headers.get("transfer-encoding", "").lower():
-                raw = b"%x\r\n%s\r\n" % (len(event.data), event.data)
-            else:
-                raw = event.data
-        elif isinstance(event, ResponseEndOfMessage):
-            if "chunked" in self.response.headers.get("transfer-encoding", "").lower():
-                raw = b"0\r\n\r\n"
-            elif http1.expected_http_body_size(self.request, self.response) == -1:
-                yield commands.CloseConnection(self.conn)
-                return
-            else:
-                raw = False
-            self.request = None
-            self.response = None
-            self.stream_id += 2
-            self.state = self.read_request_headers
-            yield from self.state(events.DataReceived(self.conn, b""))
-        else:
-            raise NotImplementedError(f"{event}")
-
-        if raw:
-            yield commands.SendData(self.conn, raw)
-
-    def read_request_headers(self, event: events.Event) -> HttpEventGenerator:
-        if isinstance(event, events.DataReceived):
-            request_head = self.buf.maybe_extract_lines()
-            if request_head:
-                request_head = [bytes(x) for x in request_head]  # TODO: Make url.parse compatible with bytearrays
-                self.request = http.HTTPRequest.wrap(http1_sansio.read_request_head(request_head))
-                yield RequestHeaders(self.request, self.stream_id)
-
-                if self.request.first_line_format == "authority":
-                    # The previous proxy server implementation tried to read the request body here:
-                    # https://github.com/mitmproxy/mitmproxy/blob/45e3ae0f9cb50b0edbf4180fd969ea99d40bdf7b/mitmproxy/proxy/protocol/http.py#L251-L255
-                    # We don't do this to be compliant with the h2 spec:
-                    # https://http2.github.io/http2-spec/#CONNECT
-                    self.state = self.wait
-                else:
-                    expected_size = http1.expected_http_body_size(self.request)
-                    self.body_reader = self.make_body_reader(expected_size)
-                    self.state = self.read_request_body
-                    yield from self.state(event)
-        elif isinstance(event, events.ConnectionClosed):
-            pass  # TODO: Better handling, tear everything down.
-        else:
-            raise ValueError(f"Unexpected event: {event}")
-
-    def read_request_body(self, event: events.Event) -> HttpEventGenerator:
-        for e in self.read_body(event, True):
-            if isinstance(e, RequestEndOfMessage):
-                self.state = self.wait
-                yield from self.state(event)
-            yield e
-
-
-class Http1Client(Http1Connection):
-    conn: Server
-    send_queue: typing.List[HttpEvent]
-    """A queue of send events for flows other than the one that is currently being transmitted."""
-
-    def __init__(self, conn: Server):
-        super().__init__(conn)
-        self.state = self.read_response_headers
-        self.send_queue = []
-
-    def send(self, event: HttpEvent) -> commands.TCommandGenerator:
-        if not self.stream_id:
-            assert isinstance(event, RequestHeaders)
-            self.stream_id = event.stream_id
-            self.request = event.request
-        if self.stream_id != event.stream_id:
-            # Assuming an h2 server, we may have multiple Streams that try to send requests
-            # over a single h1 connection. To keep things relatively simple, we don't do any HTTP/1 pipelining
-            # but keep a queue of still-to-send requests.
-            self.send_queue.append(event)
-            return
-
-        if isinstance(event, RequestHeaders):
-            raw = http1.assemble_request_head(event.request)
-        elif isinstance(event, RequestData):
-            if "chunked" in self.request.headers.get("transfer-encoding", "").lower():
-                raw = b"%x\r\n%s\r\n" % (len(event.data), event.data)
-            else:
-                raw = event.data
-        elif isinstance(event, RequestEndOfMessage):
-            if "chunked" in self.request.headers.get("transfer-encoding", "").lower():
-                raw = b"0\r\n\r\n"
-            elif http1.expected_http_body_size(self.request) == -1:
-                assert not self.send_queue
-                yield commands.CloseConnection(self.conn)
-                return
-            else:
-                raw = False
-        else:
-            raise NotImplementedError(f"{event}")
-
-        if raw:
-            yield commands.SendData(self.conn, raw)
-
-    def read_response_headers(self, event: events.ConnectionEvent) -> HttpEventGenerator:
-        if isinstance(event, events.DataReceived):
-            response_head = self.buf.maybe_extract_lines()
-
-            if response_head:
-                response_head = [bytes(x) for x in response_head]
-                self.response = http.HTTPResponse.wrap(http1_sansio.read_response_head(response_head))
-                yield ResponseHeaders(self.response, self.stream_id)
-
-                expected_size = http1.expected_http_body_size(self.request, self.response)
-                self.body_reader = self.make_body_reader(expected_size)
-
-                self.state = self.read_response_body
-                yield from self.state(event)
-        elif isinstance(event, events.ConnectionClosed):
-            if self.stream_id:
-                raise NotImplementedError(f"{event}")
-            else:
-                return
-        else:
-            raise ValueError(f"Unexpected event: {event}")
-
-    def read_response_body(self, event: events.ConnectionEvent) -> HttpEventGenerator:
-        for e in self.read_body(event, False):
-            yield e
-            if isinstance(e, ResponseEndOfMessage):
-                self.state = self.read_response_headers
-                self.stream_id = None
-                self.request = None
-                self.response = None
-                if self.send_queue:
-                    events = self.send_queue
-                    self.send_queue = []
-                    for e in events:
-                        yield from self.send(e)
-
-
-class Http2Server:
-    pass  # TODO
-
-
-class Http2Client:
-    pass  # TODO
-
-
 class HttpStream(Layer):
     request_body_buf: bytes
     response_body_buf: bytes
@@ -556,7 +257,7 @@ class HTTPLayer(Layer):
     mode: HTTPMode
     stream_by_command: typing.Dict[commands.Command, HttpStream]
     streams: typing.Dict[int, HttpStream]
-    connections: typing.Dict[Connection, typing.Union[Http1Connection, HttpStream]]
+    connections: typing.Dict[Connection, typing.Union[HttpConnection, HttpStream]]
     waiting_for_connection: typing.DefaultDict[Connection, typing.List[GetHttpConnection]]
     event_queue: typing.Deque[
         typing.Union[HttpEvent, HttpCommand, commands.Command]
@@ -714,7 +415,7 @@ class HTTPLayer(Layer):
 
     def event_to_child(
             self,
-            stream: typing.Union[Http1Connection, HttpStream],
+            stream: typing.Union[HttpConnection, HttpStream],
             event: events.Event,
     ) -> None:
         stream_events = list(stream.handle_event(event))

mitmproxy/proxy2/layers/http/http1.py

@@ -0,0 +1,253 @@
+import typing
+from abc import abstractmethod
+
+import h11
+from h11._readers import ChunkedReader, ContentLengthReader, Http10Reader
+from h11._receivebuffer import ReceiveBuffer
+
+from mitmproxy import http
+from mitmproxy.net.http import http1
+from mitmproxy.net.http.http1 import read_sansio as http1_sansio
+from mitmproxy.proxy2 import commands, events
+from mitmproxy.proxy2.context import Client, Connection, Server
+from mitmproxy.proxy2.layers.http.base import StreamId
+from .base import HttpConnection
+from .events import HttpEvent, RequestData, RequestEndOfMessage, RequestHeaders, ResponseData, ResponseEndOfMessage, \
+    ResponseHeaders
+
+TBodyReader = typing.Union[ChunkedReader, Http10Reader, ContentLengthReader]
+
+
+class Http1Connection(HttpConnection):
+    conn: Connection
+    stream_id: StreamId = None
+    request: http.HTTPRequest
+    response: http.HTTPResponse
+    state: typing.Callable[[events.Event], typing.Iterator[HttpEvent]]
+    body_reader: TBodyReader
+    buf: ReceiveBuffer
+
+    def __init__(self, conn: Connection):
+        assert isinstance(conn, Connection)
+        self.conn = conn
+        self.buf = ReceiveBuffer()
+
+    def handle_event(self, event: events.Event) -> typing.Iterator[HttpEvent]:
+        if isinstance(event, events.DataReceived):
+            self.buf += event.data
+        yield from self.state(event)
+
+    @abstractmethod
+    def send(self, event: HttpEvent) -> commands.TCommandGenerator:
+        yield from ()
+
+    def make_body_reader(self, expected_size: typing.Optional[int]) -> TBodyReader:
+        if expected_size is None:
+            return ChunkedReader()
+        elif expected_size == -1:
+            return Http10Reader()
+        else:
+            return ContentLengthReader(expected_size)
+
+    def read_body(self, event: events.Event, is_request: bool) -> typing.Iterator[HttpEvent]:
+        while True:
+            try:
+                if isinstance(event, events.DataReceived):
+                    h11_event = self.body_reader(self.buf)
+                elif isinstance(event, events.ConnectionClosed):
+                    h11_event = self.body_reader.read_eof()
+                else:
+                    raise ValueError(f"Unexpected event: {event}")
+            except h11.ProtocolError:
+                raise  # FIXME
+
+            if h11_event is None:
+                return
+            elif isinstance(h11_event, h11.Data):
+                h11_event.data: bytearray  # type checking
+                if is_request:
+                    yield RequestData(bytes(h11_event.data), self.stream_id)
+                else:
+                    yield ResponseData(bytes(h11_event.data), self.stream_id)
+            elif isinstance(h11_event, h11.EndOfMessage):
+                if is_request:
+                    yield RequestEndOfMessage(self.stream_id)
+                else:
+                    yield ResponseEndOfMessage(self.stream_id)
+                return
+
+    def wait(self, event: events.Event) -> typing.Iterator[HttpEvent]:
+        """
+        We wait for the current flow to be finished before parsing the next message,
+        as we may want to upgrade to WebSocket or plain TCP before that.
+        """
+        if isinstance(event, events.DataReceived):
+            return
+        elif isinstance(event, events.ConnectionClosed):
+            return
+        else:
+            yield from ()
+            raise ValueError(f"Unexpected event: {event}")
+
+
+class Http1Server(Http1Connection):
+    """A simple HTTP/1 server with no pipelining support."""
+    conn: Client
+
+    def __init__(self, conn: Client):
+        super().__init__(conn)
+        self.state = self.read_request_headers
+        self.stream_id = -1
+
+    def send(self, event: HttpEvent) -> commands.TCommandGenerator:
+        if isinstance(event, ResponseHeaders):
+            self.response = event.response
+            raw = http1.assemble_response_head(event.response)
+            if self.request.first_line_format == "authority":
+                assert self.state == self.wait
+                self.body_reader = self.make_body_reader(-1)
+                self.state = self.read_request_body
+                yield from self.state(events.DataReceived(self.conn, b""))
+        elif isinstance(event, ResponseData):
+            if "chunked" in self.response.headers.get("transfer-encoding", "").lower():
+                raw = b"%x\r\n%s\r\n" % (len(event.data), event.data)
+            else:
+                raw = event.data
+        elif isinstance(event, ResponseEndOfMessage):
+            if "chunked" in self.response.headers.get("transfer-encoding", "").lower():
+                raw = b"0\r\n\r\n"
+            elif http1.expected_http_body_size(self.request, self.response) == -1:
+                yield commands.CloseConnection(self.conn)
+                return
+            else:
+                raw = False
+            self.request = None
+            self.response = None
+            self.stream_id += 2
+            self.state = self.read_request_headers
+            yield from self.state(events.DataReceived(self.conn, b""))
+        else:
+            raise NotImplementedError(f"{event}")
+
+        if raw:
+            yield commands.SendData(self.conn, raw)
+
+    def read_request_headers(self, event: events.Event) -> typing.Iterator[HttpEvent]:
+        if isinstance(event, events.DataReceived):
+            request_head = self.buf.maybe_extract_lines()
+            if request_head:
+                request_head = [bytes(x) for x in request_head]  # TODO: Make url.parse compatible with bytearrays
+                self.request = http.HTTPRequest.wrap(http1_sansio.read_request_head(request_head))
+                yield RequestHeaders(self.request, self.stream_id)
+
+                if self.request.first_line_format == "authority":
+                    # The previous proxy server implementation tried to read the request body here:
+                    # https://github.com/mitmproxy/mitmproxy/blob/45e3ae0f9cb50b0edbf4180fd969ea99d40bdf7b/mitmproxy/proxy/protocol/http.py#L251-L255
+                    # We don't do this to be compliant with the h2 spec:
+                    # https://http2.github.io/http2-spec/#CONNECT
+                    self.state = self.wait
+                else:
+                    expected_size = http1.expected_http_body_size(self.request)
+                    self.body_reader = self.make_body_reader(expected_size)
+                    self.state = self.read_request_body
+                    yield from self.state(event)
+        elif isinstance(event, events.ConnectionClosed):
+            pass  # TODO: Better handling, tear everything down.
+        else:
+            raise ValueError(f"Unexpected event: {event}")
+
+    def read_request_body(self, event: events.Event) -> typing.Iterator[HttpEvent]:
+        for e in self.read_body(event, True):
+            if isinstance(e, RequestEndOfMessage):
+                self.state = self.wait
+                yield from self.state(event)
+            yield e
+
+
+class Http1Client(Http1Connection):
+    conn: Server
+    send_queue: typing.List[HttpEvent]
+    """A queue of send events for flows other than the one that is currently being transmitted."""
+
+    def __init__(self, conn: Server):
+        super().__init__(conn)
+        self.state = self.read_response_headers
+        self.send_queue = []
+
+    def send(self, event: HttpEvent) -> commands.TCommandGenerator:
+        if not self.stream_id:
+            assert isinstance(event, RequestHeaders)
+            self.stream_id = event.stream_id
+            self.request = event.request
+        if self.stream_id != event.stream_id:
+            # Assuming an h2 server, we may have multiple Streams that try to send requests
+            # over a single h1 connection. To keep things relatively simple, we don't do any HTTP/1 pipelining
+            # but keep a queue of still-to-send requests.
+            self.send_queue.append(event)
+            return
+
+        if isinstance(event, RequestHeaders):
+            raw = http1.assemble_request_head(event.request)
+        elif isinstance(event, RequestData):
+            if "chunked" in self.request.headers.get("transfer-encoding", "").lower():
+                raw = b"%x\r\n%s\r\n" % (len(event.data), event.data)
+            else:
+                raw = event.data
+        elif isinstance(event, RequestEndOfMessage):
+            if "chunked" in self.request.headers.get("transfer-encoding", "").lower():
+                raw = b"0\r\n\r\n"
+            elif http1.expected_http_body_size(self.request) == -1:
+                assert not self.send_queue
+                yield commands.CloseConnection(self.conn)
+                return
+            else:
+                raw = False
+        else:
+            raise NotImplementedError(f"{event}")
+
+        if raw:
+            yield commands.SendData(self.conn, raw)
+
+    def read_response_headers(self, event: events.Event) -> typing.Iterator[HttpEvent]:
+        assert isinstance(event, events.ConnectionEvent)
+        if isinstance(event, events.DataReceived):
+            response_head = self.buf.maybe_extract_lines()
+
+            if response_head:
+                response_head = [bytes(x) for x in response_head]
+                self.response = http.HTTPResponse.wrap(http1_sansio.read_response_head(response_head))
+                yield ResponseHeaders(self.response, self.stream_id)
+
+                expected_size = http1.expected_http_body_size(self.request, self.response)
+                self.body_reader = self.make_body_reader(expected_size)
+
+                self.state = self.read_response_body
+                yield from self.state(event)
+        elif isinstance(event, events.ConnectionClosed):
+            if self.stream_id:
+                raise NotImplementedError(f"{event}")
+            else:
+                return
+        else:
+            raise ValueError(f"Unexpected event: {event}")
+
+    def read_response_body(self, event: events.Event) -> typing.Iterator[HttpEvent]:
+        assert isinstance(event, events.ConnectionEvent)
+        for e in self.read_body(event, False):
+            yield e
+            if isinstance(e, ResponseEndOfMessage):
+                self.state = self.read_response_headers
+                self.stream_id = None
+                self.request = None
+                self.response = None
+                if self.send_queue:
+                    send_queue = self.send_queue
+                    self.send_queue = []
+                    for ev in send_queue:
+                        yield from self.send(ev)
+
+
+__all__ = [
+    "Http1Client",
+    "Http1Server",
+]

mitmproxy/proxy2/layers/http/http2.py

@@ -0,0 +1,12 @@
+class Http2Server:
+    pass  # TODO
+
+
+class Http2Client:
+    pass  # TODO
+
+
+__all__ = [
+    "Http2Client",
+    "Http2Server",
+]

test/mitmproxy/proxy2/layers/test_http.py

@@ -4,8 +4,8 @@ from mitmproxy.http import HTTPResponse
 from mitmproxy.proxy.protocol.http import HTTPMode
 from mitmproxy.proxy2.commands import Hook, OpenConnection, SendData
 from mitmproxy.proxy2.events import ConnectionClosed, DataReceived
+from mitmproxy.proxy2 import layers
 from mitmproxy.proxy2.layers import tls
-from mitmproxy.proxy2.layers.http import http
 from test.mitmproxy.proxy2.tutils import Placeholder, Playbook, reply, reply_establish_server_tls, reply_next_layer
 
 
@@ -14,7 +14,7 @@ def test_http_proxy(tctx):
     server = Placeholder()
     flow = Placeholder()
     assert (
-            Playbook(http.HTTPLayer(tctx, HTTPMode.regular))
+            Playbook(layers.HTTPLayer(tctx, HTTPMode.regular))
             >> DataReceived(tctx.client, b"GET http://example.com/foo?hello=1 HTTP/1.1\r\nHost: example.com\r\n\r\n")
             << Hook("requestheaders", flow)
             >> reply()
@@ -39,7 +39,7 @@ def test_https_proxy(strategy, tctx):
     """Test a CONNECT request, followed by a HTTP GET /"""
     server = Placeholder()
     flow = Placeholder()
-    playbook = Playbook(http.HTTPLayer(tctx, HTTPMode.regular))
+    playbook = Playbook(layers.HTTPLayer(tctx, HTTPMode.regular))
     tctx.options.connection_strategy = strategy
 
     (playbook
@@ -54,7 +54,7 @@ def test_https_proxy(strategy, tctx):
      << SendData(tctx.client, b'HTTP/1.1 200 Connection established\r\n\r\n')
      >> DataReceived(tctx.client, b"GET /foo?hello=1 HTTP/1.1\r\nHost: example.com\r\n\r\n")
      << Hook("next_layer", Placeholder())
-     >> reply_next_layer(lambda ctx: http.HTTPLayer(ctx, HTTPMode.transparent))
+     >> reply_next_layer(lambda ctx: layers.HTTPLayer(ctx, HTTPMode.transparent))
      << Hook("requestheaders", flow)
      >> reply()
      << Hook("request", flow)
@@ -82,7 +82,7 @@ def test_redirect(strategy, https_server, https_client, tctx):
     server = Placeholder()
     flow = Placeholder()
     tctx.options.connection_strategy = strategy
-    p = Playbook(http.HTTPLayer(tctx, HTTPMode.regular), hooks=False)
+    p = Playbook(layers.HTTPLayer(tctx, HTTPMode.regular), hooks=False)
 
     def redirect(hook: Hook):
         if https_server:
@@ -98,7 +98,7 @@ def test_redirect(strategy, https_server, https_client, tctx):
         p << SendData(tctx.client, b'HTTP/1.1 200 Connection established\r\n\r\n')
         p >> DataReceived(tctx.client, b"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n")
         p << Hook("next_layer", Placeholder())
-        p >> reply_next_layer(lambda ctx: http.HTTPLayer(ctx, HTTPMode.transparent))
+        p >> reply_next_layer(lambda ctx: layers.HTTPLayer(ctx, HTTPMode.transparent))
     else:
         p >> DataReceived(tctx.client, b"GET http://example.com/ HTTP/1.1\r\nHost: example.com\r\n\r\n")
     p << Hook("request", flow)
@@ -131,7 +131,7 @@ def test_multiple_server_connections(tctx):
         return side_effect
 
     assert (
-            Playbook(http.HTTPLayer(tctx, HTTPMode.regular), hooks=False)
+            Playbook(layers.HTTPLayer(tctx, HTTPMode.regular), hooks=False)
             >> DataReceived(tctx.client, b"GET http://example.com/ HTTP/1.1\r\nHost: example.com\r\n\r\n")
             << Hook("request", Placeholder())
             >> reply(side_effect=redirect("http://one.redirect/"))
@@ -160,7 +160,7 @@ def test_http_reply_from_proxy(tctx):
         hook.data.response = HTTPResponse.make(418)
 
     assert (
-            Playbook(http.HTTPLayer(tctx, HTTPMode.regular), hooks=False)
+            Playbook(layers.HTTPLayer(tctx, HTTPMode.regular), hooks=False)
             >> DataReceived(tctx.client, b"GET http://example.com/ HTTP/1.1\r\nHost: example.com\r\n\r\n")
             << Hook("request", Placeholder())
             >> reply(side_effect=reply_from_proxy)
@@ -177,7 +177,7 @@ def test_disconnect_while_intercept(tctx):
     flow = Placeholder()
 
     assert (
-            Playbook(http.HTTPLayer(tctx, HTTPMode.regular), hooks=False)
+            Playbook(layers.HTTPLayer(tctx, HTTPMode.regular), hooks=False)
             >> DataReceived(tctx.client, b"CONNECT example.com:80 HTTP/1.1\r\n\r\n")
             << Hook("http_connect", Placeholder())
             >> reply()
@@ -186,7 +186,7 @@ def test_disconnect_while_intercept(tctx):
             << SendData(tctx.client, b'HTTP/1.1 200 Connection established\r\n\r\n')
             >> DataReceived(tctx.client, b"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n")
             << Hook("next_layer", Placeholder())
-            >> reply_next_layer(lambda ctx: http.HTTPLayer(ctx, HTTPMode.transparent))
+            >> reply_next_layer(lambda ctx: layers.HTTPLayer(ctx, HTTPMode.transparent))
             << Hook("request", flow)
             >> ConnectionClosed(server1)
             >> reply(to=-2)