From d68c364b358238cb3e3410de2f147571bcc8ee5c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 4 Feb 2021 02:59:01 +0100 Subject: [PATCH] inline/move http.make_* functions --- mitmproxy/addons/proxyauth.py | 25 +++--- mitmproxy/http.py | 80 +++---------------- mitmproxy/proxy/layers/http/__init__.py | 13 ++- mitmproxy/proxy/layers/http/_http1.py | 50 +++++++++--- .../proxy/layers/http/_upstream_proxy.py | 16 +++- .../mitmproxy/proxy/layers/http/test_http1.py | 5 ++ test/mitmproxy/test_http.py | 24 ------ 7 files changed, 99 insertions(+), 114 deletions(-) diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py index 04d52a0ac..e949aa4d8 100644 --- a/mitmproxy/addons/proxyauth.py +++ b/mitmproxy/addons/proxyauth.py @@ -7,7 +7,6 @@ from typing import Tuple import ldap3 import passlib.apache -import mitmproxy.net.http from mitmproxy import ctx from mitmproxy import exceptions from mitmproxy import http @@ -83,15 +82,23 @@ class ProxyAuth: def auth_required_response(self) -> http.Response: if self.is_proxy_auth(): - return http.make_error_response( - status_codes.PROXY_AUTH_REQUIRED, - headers=mitmproxy.http.Headers(Proxy_Authenticate=f'Basic realm="{REALM}"'), - ) + status_code = status_codes.PROXY_AUTH_REQUIRED + headers = {"Proxy-Authenticate": f'Basic realm="{REALM}"'} else: - return http.make_error_response( - status_codes.UNAUTHORIZED, - headers=mitmproxy.http.Headers(WWW_Authenticate=f'Basic realm="{REALM}"'), - ) + status_code = status_codes.UNAUTHORIZED + headers = {"WWW-Authenticate": f'Basic realm="{REALM}"'} + + reason = http.status_codes.RESPONSES[status_code] + return http.Response.make( + status_code, + ( + f"" + f"{status_code} {reason}" + f"

{status_code} {reason}

" + f"" + ), + headers + ) def check(self, f: http.HTTPFlow) -> Optional[Tuple[str, str]]: """ diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 420fcbdb5..577685d0e 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -1,4 +1,4 @@ -import html +import re import re import time import urllib.parse @@ -13,14 +13,13 @@ from typing import Optional from typing import Tuple from typing import Union -from mitmproxy.net.http import url from mitmproxy import flow -from mitmproxy import version from mitmproxy.coretypes import multidict from mitmproxy.coretypes import serializable -from mitmproxy.net import http, encoding +from mitmproxy.net import encoding from mitmproxy.net.http import cookies, multipart from mitmproxy.net.http import status_codes +from mitmproxy.net.http import url from mitmproxy.net.http.headers import assemble_content_type, parse_content_type from mitmproxy.proxy import context from mitmproxy.utils import human @@ -1158,69 +1157,10 @@ class HTTPFlow(flow.Flow): return f -def make_error_response( - status_code: int, - message: str = "", - headers: Optional[Headers] = None, -) -> Response: - body: bytes = """ - - - {status_code} {reason} - - -

{status_code} {reason}

-

{message}

- - - """.strip().format( - status_code=status_code, - reason=http.status_codes.RESPONSES.get(status_code, "Unknown"), - message=html.escape(message), - ).encode("utf8", "replace") - - if not headers: - headers = Headers( - Server=version.MITMPROXY, - Connection="close", - Content_Length=str(len(body)), - Content_Type="text/html" - ) - - return Response.make(status_code, body, headers) - - -def make_connect_request(address: Tuple[str, int]) -> Request: - return Request( - host=address[0], - port=address[1], - method=b"CONNECT", - scheme=b"", - authority=f"{address[0]}:{address[1]}".encode(), - path=b"", - http_version=b"HTTP/1.1", - headers=Headers(), - content=b"", - trailers=None, - timestamp_start=time.time(), - timestamp_end=time.time(), - ) - - -def make_connect_response(http_version): - # Do not send any response headers as it breaks proxying non-80 ports on - # Android emulators using the -http-proxy option. - return Response( - http_version, - 200, - b"Connection established", - Headers(), - b"", - None, - time.time(), - time.time(), - ) - - -def make_expect_continue_response(): - return Response.make(100) +__all__ = [ + "HTTPFlow", + "Message", + "Request", + "Response", + "Headers", +] diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index e45b5ac86..84211fca7 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -449,7 +449,18 @@ class HttpStream(layer.Layer): def handle_connect_finish(self): if not self.flow.response: - self.flow.response = http.make_connect_response(self.flow.request.data.http_version) + # Do not send any response headers as it breaks proxying non-80 ports on + # Android emulators using the -http-proxy option. + self.flow.response = http.Response( + self.flow.request.data.http_version, + 200, + b"Connection established", + http.Headers(), + b"", + None, + time.time(), + time.time(), + ) if 200 <= self.flow.response.status_code < 300: yield SendHttp(ResponseHeaders(self.stream_id, self.flow.response), self.context.client) diff --git a/mitmproxy/proxy/layers/http/_http1.py b/mitmproxy/proxy/layers/http/_http1.py index afaa92513..14da52ab8 100644 --- a/mitmproxy/proxy/layers/http/_http1.py +++ b/mitmproxy/proxy/layers/http/_http1.py @@ -1,11 +1,12 @@ import abc +import html from typing import Union, Optional, Callable, Type import h11 from h11._readers import ChunkedReader, ContentLengthReader, Http10Reader from h11._receivebuffer import ReceiveBuffer -from mitmproxy import http +from mitmproxy import http, version from mitmproxy.net.http import http1, status_codes from mitmproxy.proxy import commands, events, layer from mitmproxy.proxy.context import Connection, ConnectionState, Context @@ -145,13 +146,13 @@ class Http1Connection(HttpConnection, metaclass=abc.ABCMeta): yield from self.make_pipe() return connection_done = ( - http1.expected_http_body_size(self.request, self.response) == -1 - or http1.connection_close(self.request.http_version, self.request.headers) - or http1.connection_close(self.response.http_version, self.response.headers) - # If we proxy HTTP/2 to HTTP/1, we only use upstream connections for one request. - # This simplifies our connection management quite a bit as we can rely on - # the proxyserver's max-connection-per-server throttling. - or (self.request.is_http2 and isinstance(self, Http1Client)) + http1.expected_http_body_size(self.request, self.response) == -1 + or http1.connection_close(self.request.http_version, self.request.headers) + or http1.connection_close(self.response.http_version, self.response.headers) + # If we proxy HTTP/2 to HTTP/1, we only use upstream connections for one request. + # This simplifies our connection management quite a bit as we can rely on + # the proxyserver's max-connection-per-server throttling. + or (self.request.is_http2 and isinstance(self, Http1Client)) ) if connection_done: yield commands.CloseConnection(self.conn) @@ -211,7 +212,7 @@ class Http1Server(Http1Connection): yield from self.mark_done(response=True) elif isinstance(event, ResponseProtocolError): if not self.response: - resp = http.make_error_response(event.code, event.message) + resp = make_error_response(event.code, event.message) raw = http1.assemble_response(resp) yield commands.SendData(self.conn, raw) yield commands.CloseConnection(self.conn) @@ -362,6 +363,37 @@ def make_body_reader(expected_size: Optional[int]) -> TBodyReader: return ContentLengthReader(expected_size) +def make_error_response( + status_code: int, + message: str = "", +) -> http.Response: + body: bytes = """ + + + {status_code} {reason} + + +

{status_code} {reason}

+

{message}

+ + + """.strip().format( + status_code=status_code, + reason=http.status_codes.RESPONSES.get(status_code, "Unknown"), + message=html.escape(message), + ).encode("utf8", "replace") + + return http.Response.make( + status_code, + body, + http.Headers( + Server=version.MITMPROXY, + Connection="close", + Content_Type="text/html", + ) + ) + + __all__ = [ "Http1Client", "Http1Server", diff --git a/mitmproxy/proxy/layers/http/_upstream_proxy.py b/mitmproxy/proxy/layers/http/_upstream_proxy.py index 58a3af973..5260e5f08 100644 --- a/mitmproxy/proxy/layers/http/_upstream_proxy.py +++ b/mitmproxy/proxy/layers/http/_upstream_proxy.py @@ -1,3 +1,4 @@ +import time from typing import Optional, Tuple from h11._receivebuffer import ReceiveBuffer @@ -44,7 +45,20 @@ class HttpUpstreamProxy(tunnel.TunnelLayer): if not self.send_connect: return (yield from super().start_handshake()) assert self.conn.address - req = http.make_connect_request(self.conn.address) + req = http.Request( + host=self.conn.address[0], + port=self.conn.address[1], + method=b"CONNECT", + scheme=b"", + authority=f"{self.conn.address[0]}:{self.conn.address[1]}".encode(), + path=b"", + http_version=b"HTTP/1.1", + headers=http.Headers(), + content=b"", + trailers=None, + timestamp_start=time.time(), + timestamp_end=time.time(), + ) raw = http1.assemble_request(req) yield commands.SendData(self.tunnel_connection, raw) diff --git a/test/mitmproxy/proxy/layers/http/test_http1.py b/test/mitmproxy/proxy/layers/http/test_http1.py index 26c6ea0f5..b1912bf65 100644 --- a/test/mitmproxy/proxy/layers/http/test_http1.py +++ b/test/mitmproxy/proxy/layers/http/test_http1.py @@ -5,6 +5,7 @@ from mitmproxy.proxy.commands import SendData from mitmproxy.proxy.events import DataReceived from mitmproxy.proxy.layers.http import Http1Server, ReceiveHttp, RequestHeaders, RequestEndOfMessage, \ ResponseHeaders, ResponseEndOfMessage, RequestData, Http1Client, ResponseData +from mitmproxy.proxy.layers.http._http1 import make_error_response from test.mitmproxy.proxy.tutils import Placeholder, Playbook @@ -199,3 +200,7 @@ class TestClient: >> RequestHeaders(3, req, True) << SendData(tctx.server, Placeholder(bytes)) ) + + +def test_make_error_response(): + assert make_error_response(543, 'foobar') diff --git a/test/mitmproxy/test_http.py b/test/mitmproxy/test_http.py index 2e7d49de3..3a1c7e4b5 100644 --- a/test/mitmproxy/test_http.py +++ b/test/mitmproxy/test_http.py @@ -197,27 +197,3 @@ class TestHTTPFlow: def test_timestamp_start(self): f = tflow.tflow() assert f.timestamp_start == f.request.timestamp_start - - -def test_make_error_response(): - resp = http.make_error_response(543, 'foobar', Headers()) - assert resp - - -def test_make_connect_request(): - req = http.make_connect_request(('invalidhost', 1234)) - assert req.first_line_format == 'authority' - assert req.method == 'CONNECT' - assert req.http_version == 'HTTP/1.1' - - -def test_make_connect_response(): - resp = http.make_connect_response('foobar') - assert resp.http_version == 'foobar' - assert resp.status_code == 200 - - -def test_expect_continue_response(): - resp = http.make_expect_continue_response() - assert resp.http_version == 'HTTP/1.1' - assert resp.status_code == 100