From 67a93239f48164335893e9c4542c62c9c7955db8 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 1 Jan 2021 18:10:16 +0100 Subject: [PATCH 1/3] tlsconfig: respect add_upstream_certs_to_client_chain --- mitmproxy/addons/tlsconfig.py | 8 +++++++- test/mitmproxy/addons/test_tlsconfig.py | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index f1dc90c07..766643504 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -123,6 +123,12 @@ class TlsConfig: # don't assign to client.cipher_list, doesn't need to be stored. cipher_list = client.cipher_list or DEFAULT_CIPHERS + if ctx.options.add_upstream_certs_to_client_chain: # pragma: no cover + # exempted from coverage until https://bugs.python.org/issue18233 is fixed. + extra_chain_certs = server.certificate_list + else: + extra_chain_certs = [] + ssl_ctx = net_tls.create_client_proxy_context( min_version=net_tls.Version[ctx.options.tls_version_client_min], max_version=net_tls.Version[ctx.options.tls_version_client_max], @@ -132,7 +138,7 @@ class TlsConfig: chain_file=entry.chain_file, request_client_cert=False, alpn_select_callback=alpn_select_callback, - extra_chain_certs=server.certificate_list, + extra_chain_certs=extra_chain_certs, dhparams=self.certstore.dhparams, ) tls_start.ssl_conn = SSL.Connection(ssl_ctx) diff --git a/test/mitmproxy/addons/test_tlsconfig.py b/test/mitmproxy/addons/test_tlsconfig.py index 26d0a491b..de318583a 100644 --- a/test/mitmproxy/addons/test_tlsconfig.py +++ b/test/mitmproxy/addons/test_tlsconfig.py @@ -114,7 +114,6 @@ class TestTlsConfig: ciphers_client="ECDHE-ECDSA-AES128-GCM-SHA256", ) ctx = context.Context(context.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options) - tctx.options.add_upstream_certs_to_client_chain = True tls_start = tls.TlsStartData(ctx.client, context=ctx) ta.tls_start(tls_start) From 455fee1126ba93f17d475b075d41067793646340 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 1 Jan 2021 18:15:58 +0100 Subject: [PATCH 2/3] warn about nonfunctioning options, permanently remove unused ones. If you are affected by this change, please do reach out by filing an issue. --- CHANGELOG.md | 3 +++ mitmproxy/addons/core.py | 4 ++++ mitmproxy/addons/streambodies.py | 15 ------------- mitmproxy/addons/upstream_auth.py | 3 +++ mitmproxy/options.py | 25 ---------------------- mitmproxy/utils/arg_check.py | 1 - test/mitmproxy/addons/test_streambodies.py | 6 ------ 7 files changed, 10 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 190b604cb..6cecf7f73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,9 @@ If you depend on these features, please raise your voice in * Remove all deprecated pathod and pathoc tools and modules (@Kriechi) * In reverse proxy mode, mitmproxy now does not assume TLS if no scheme is given but a custom port is provided (@mhils) +* Remove the following options: `http2_priority`, `relax_http_form_validation`, `upstream_bind_address`, + `spoof_source_address`, and `stream_websockets`. If you depended on one of them please let us know. + mitmproxy never phones home, which means we don't know how prominently these options were used. (@mhils) * --- TODO: add new PRs above this line --- * ... and various other fixes, documentation improvements, dependency version bumps, etc. diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index 6aa5856db..2034947aa 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -42,6 +42,10 @@ class Core: "add_upstream_certs_to_client_chain requires the upstream_cert option to be enabled." ) if "body_size_limit" in updated: + if opts.body_size_limit: # pragma: no cover + ctx.log.warn( + "body_size_limit is currently nonfunctioning, " + "see https://github.com/mitmproxy/mitmproxy/issues/4348") try: human.parse_size(opts.body_size_limit) except ValueError: diff --git a/mitmproxy/addons/streambodies.py b/mitmproxy/addons/streambodies.py index c568e944c..e0cd8d7a3 100644 --- a/mitmproxy/addons/streambodies.py +++ b/mitmproxy/addons/streambodies.py @@ -19,13 +19,6 @@ class StreamBodies: Understands k/m/g suffixes, i.e. 3m for 3 megabytes. """ ) - loader.add_option( - "stream_websockets", bool, False, - """ - Stream WebSocket messages between client and server. - Messages are captured and cannot be modified. - """ - ) def configure(self, updated): if "stream_large_bodies" in updated and ctx.options.stream_large_bodies: @@ -54,11 +47,3 @@ class StreamBodies: def responseheaders(self, f): self.run(f, False) - - def websocket_start(self, f): - if ctx.options.stream_websockets: - f.stream = True - ctx.log.info("Streaming WebSocket messages between {client} and {server}".format( - client=human.format_address(f.client_conn.peername), - server=human.format_address(f.server_conn.address)) - ) diff --git a/mitmproxy/addons/upstream_auth.py b/mitmproxy/addons/upstream_auth.py index c0581e27c..dc1a64419 100644 --- a/mitmproxy/addons/upstream_auth.py +++ b/mitmproxy/addons/upstream_auth.py @@ -47,6 +47,9 @@ class UpstreamAuth(): if ctx.options.upstream_auth is None: self.auth = None else: + if ctx.options.upstream_auth: # pragma: no cover + ctx.log.warn("upstream_auth is currently nonfunctioning, " + "see https://github.com/mitmproxy/mitmproxy/issues/4348") self.auth = parse_upstream_auth(ctx.options.upstream_auth) def http_connect(self, f): diff --git a/mitmproxy/options.py b/mitmproxy/options.py index a047358bb..b480045ec 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -83,10 +83,6 @@ class Options(optmanager.OptManager): "listen_port", int, LISTEN_PORT, "Proxy service port." ) - self.add_option( - "upstream_bind_address", str, "", - "Address to bind upstream requests to." - ) self.add_option( "mode", str, "regular", """ @@ -105,13 +101,6 @@ class Options(optmanager.OptManager): "Enable/disable HTTP/2 support. " "HTTP/2 support is enabled by default.", ) - self.add_option( - "http2_priority", bool, False, - """ - PRIORITY forwarding for HTTP/2 connections. Disabled by default to ensure compatibility - with misbehaving servers. - """ - ) self.add_option( "websocket", bool, True, "Enable/disable WebSocket support. " @@ -122,14 +111,6 @@ class Options(optmanager.OptManager): "Enable/disable raw TCP connections. " "TCP connections are enabled by default. " ) - - self.add_option( - "spoof_source_address", bool, False, - """ - Use the client's IP for server-side connections. Combine with - --upstream-bind-address to spoof a fixed source address. - """ - ) self.add_option( "ssl_insecure", bool, False, "Do not verify upstream server SSL/TLS certificates." @@ -166,11 +147,5 @@ class Options(optmanager.OptManager): TLS key size for certificates and CA. """ ) - self.add_option( - "relax_http_form_validation", bool, False, - """ - Disable HTTP form validation. - """ - ) self.update(**kwargs) diff --git a/mitmproxy/utils/arg_check.py b/mitmproxy/utils/arg_check.py index a0089ee63..44ef971ac 100644 --- a/mitmproxy/utils/arg_check.py +++ b/mitmproxy/utils/arg_check.py @@ -71,7 +71,6 @@ REPLACEMENTS = { "--order": "view_order", "--no-mouse": "console_mouse", "--reverse": "view_order_reversed", - "--no-http2-priority": "http2_priority", "--no-websocket": "websocket", "--no-upstream-cert": "upstream_cert", "--upstream-trusted-confdir": "ssl_verify_upstream_trusted_confdir", diff --git a/test/mitmproxy/addons/test_streambodies.py b/test/mitmproxy/addons/test_streambodies.py index b980ea0b7..d62f869f1 100644 --- a/test/mitmproxy/addons/test_streambodies.py +++ b/test/mitmproxy/addons/test_streambodies.py @@ -29,9 +29,3 @@ def test_simple(): f = tflow.tflow(resp=True) f.response.headers["content-length"] = "invalid" tctx.cycle(sa, f) - - tctx.configure(sa, stream_websockets = True) - f = tflow.twebsocketflow() - assert not f.stream - sa.websocket_start(f) - assert f.stream From 7941e09016162db083955f9f5a1367980f3a52a9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 1 Jan 2021 22:45:05 +0100 Subject: [PATCH 3/3] docs: update protocols section --- docs/src/assets/style.scss | 2 +- docs/src/content/concepts-protocols.md | 102 +++++++++++-------------- 2 files changed, 44 insertions(+), 60 deletions(-) diff --git a/docs/src/assets/style.scss b/docs/src/assets/style.scss index 7846ae426..e5254b541 100644 --- a/docs/src/assets/style.scss +++ b/docs/src/assets/style.scss @@ -38,7 +38,7 @@ body > div { } #main { - padding: 3rem; + padding: 3rem 3rem 9rem; } .example { diff --git a/docs/src/content/concepts-protocols.md b/docs/src/content/concepts-protocols.md index 9a3dd6e48..f15caa6f7 100644 --- a/docs/src/content/concepts-protocols.md +++ b/docs/src/content/concepts-protocols.md @@ -7,85 +7,69 @@ menu: # Protocols -## HTTP/1.0 and HTTP/1.1 +mitmproxy not only supports HTTP, but also other important web protocols. +This page lists details and known limitations of the respective protocol implementations. +Most protocols can be disabled by toggling the respective [option]({{< relref concepts-options >}}). -[RFC7230: HTTP/1.1: Message Syntax and Routing](http://tools.ietf.org/html/rfc7230) +## HTTP/1.x -[RFC7231: HTTP/1.1: Semantics and Content](http://tools.ietf.org/html/rfc7231) +HTTP/1.0 and HTTP/1.1 support in mitmproxy is based on our custom HTTP stack, which is particularly robust to HTTP syntax +errors. Protocol violations are often deliberately forwarded or inserted at the proxy. -HTTP/1.0 and HTTP/1.1 support in mitmproxy is based on our custom HTTP stack, -which takes care of all semantics and on-the-wire parsing/serialization tasks. +##### Known Limitations -mitmproxy currently does not support parsing HTTP trailers - but if you want to send -us a PR, we promise to take look! +- Trailers: mitmproxy currently does not support HTTP trailers, but we are happy to accept contributions. + +##### RFCs + +- [RFC7230: HTTP/1.1: Message Syntax and Routing](http://tools.ietf.org/html/rfc7230) +- [RFC7231: HTTP/1.1: Semantics and Content](http://tools.ietf.org/html/rfc7231) ## HTTP/2 -[RFC7540: Hypertext Transfer Protocol Version 2 (HTTP/2)](http://tools.ietf.org/html/rfc7540>) +HTTP/2 support in mitmproxy is based on [hyper-h2](https://github.com/python-hyper/hyper-h2). In case the upstream +server does not speak HTTP/2, mitmproxy seamlessly translates messages to HTTP/1. -HTTP/2 support in mitmproxy is based on -[hyper-h2](https://github.com/python-hyper/hyper-h2). It fully encapsulates the -internal state of HTTP/2 connections and provides an easy-to-use event-based -API. mitmproxy supports the majority of HTTP/2 feature and tries to -transparently pass-through as much information as possible. +##### Known Limitations -mitmproxy currently does not support HTTP/2 Cleartext (h2c) since none of the -major browser vendors have implemented it. +- *Trailers*: mitmproxy currently does not support HTTP trailers, but we are happy to accept contributions. +- *Priority Information*: mitmproxy currently ignores HTTP/2 PRIORITY frames. This does not affect the transmitted + contents, but potentially affects the order in which messages are sent. +- *Push Promises*: mitmproxy currently does not advertise support for HTTP/2 Push Promises. +- *Cleartext HTTP/2*: mitmproxy currently does not support unencrypted HTTP/2 (h2c). -Some websites are still having problems with correct HTTP/2 support in their -webservers and can cause errors, dropped connections, or simply no response at -all. We are trying to be as tolerant and forgiving as possible with the types of -data we send and receive, but -[some](https://github.com/mitmproxy/mitmproxy/issues/1745) -[faulty](https://github.com/mitmproxy/mitmproxy/issues/2823) -[implementations](https://github.com/mitmproxy/mitmproxy/issues/1824) -[simply](https://github.com/mitmproxy/mitmproxy/issues/1891) don't work well -with mitmproxy. +##### RFCs -In order to increase the compatibility of mitmproxy with HTTP/2 webservers, we -default to NOT forward any priority information that is sent by a client. You -can enable it with: `http2_priority=true`. +- [RFC7540: Hypertext Transfer Protocol Version 2 (HTTP/2)](http://tools.ietf.org/html/rfc7540) ## WebSocket -[RFC6455: The WebSocket Protocol](http://tools.ietf.org/html/rfc6455) +WebSocket support in mitmproxy is based on [wsproto](https://github.com/python-hyper/wsproto) project, including support +for message compression. -[RFC7692: Compression Extensions for WebSocket](http://tools.ietf.org/html/rfc7692) +##### Known Limitations -WebSocket support in mitmproxy is based on [wsproto](https://github.com/python-hyper/wsproto) project. It fully encapsulates -WebSocket frames/messages/connections and provides an easy-to-use event-based -API. +- *User Interface*: WebSocket messages are currently logged to the event log, but not displayed in the console or web + interface. We would welcome contributions that fix this issue. +- *Replay*: Client or server replay is not possible yet. +- *Ping*: mitmproxy will forward PING and PONG frames, but not store them. The payload is only logged to the event log. +- *Unknown Extensions*: Unknown WebSocket extensions will cause a warning message to be logged, but are otherwise passed + through as-is. This may lead to noncompliant behavior. -mitmproxy fully supports the compression extension for WebSocket messages, -provided by wsproto. Message contents are automatically compressed and -decompressed before firing events. +##### RFCs -mitmproxy currently does not display WebSocket messages in the console or web -UI. Only the WebSocket handshake flow is shown, which contains a reference to -the parent flow for all messages exchanged over this connection. +- [RFC6455: The WebSocket Protocol](http://tools.ietf.org/html/rfc6455) +- [RFC7692: Compression Extensions for WebSocket](http://tools.ietf.org/html/rfc7692) -If an endpoint sends a PING to mitmproxy, a PONG will be sent back immediately -(with the same payload if present). To keep the other connection alive, a new -PING (without a payload) is sent to the other endpoint. Unsolicited PONG's are -not forwarded. All PING's and PONG's are logged (with payload if present). +## Generic TCP Proxy -Please note that message interception, modification or replay are not possible yet. +Mitmproxy can also act as a generic TCP proxy. In this mode, mitmproxy will still detect the presence of TLS at the +beginning of a connection and perform a man-in-the-middle attack if necessary, but otherwise forward messages +unmodified. -## Raw TCP / TCP Proxy / Fallback +Users can explicitly opt into generic TCP proxying by setting the [`tcp_hosts` option]({{< relref concepts-options >}}). -In case mitmproxy does not handle a specific protocol, you can exempt -hostnames from processing, so that mitmproxy acts as a generic TCP forwarder. -This feature is closely related to the *passthrough* functionality, -but differs in two important aspects: +##### Known Limitations - * The raw TCP messages are printed to the event log. - * SSL connections will be intercepted. - -Please note that message interception, modification or replay are not possible yet. If -you are not interested in the raw TCP messages, you should use the ignore -domains feature. - -| | | -| ------------------ | ------------------ | -| command-line alias | `--tcp HOST` | -| mitmproxy shortcut | press `O` then `T` | +- *Replay*: Client or server replay is not possible yet. +- *Opportunistic TLS*: mitmproxy will not detect when a plaintext protocol upgrades to TLS (STARTTLS).