From 7d2525b4c72269318539c23a206cd4ce8bbc9575 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 3 Aug 2021 16:31:16 +0200 Subject: [PATCH 1/5] simplify upstream_cert logic --- mitmproxy/addons/tlsconfig.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index a9cc1cb92..0c8ebf92a 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -106,17 +106,10 @@ class TlsConfig: def tls_clienthello(self, tls_clienthello: tls.ClientHelloData): conn_context = tls_clienthello.context - only_non_http_alpns = ( - conn_context.client.alpn_offers and - all(x not in tls.HTTP_ALPNS for x in conn_context.client.alpn_offers) - ) tls_clienthello.establish_server_tls_first = conn_context.server.tls and ( ctx.options.connection_strategy == "eager" or ctx.options.add_upstream_certs_to_client_chain or - ctx.options.upstream_cert and ( - only_non_http_alpns or - not conn_context.client.sni - ) + ctx.options.upstream_cert ) def tls_start_client(self, tls_start: tls.TlsStartData) -> None: @@ -288,7 +281,7 @@ class TlsConfig: organization: Optional[str] = None # Use upstream certificate if available. - if conn_context.server.certificate_list: + if ctx.options.upstream_cert and conn_context.server.certificate_list: upstream_cert = conn_context.server.certificate_list[0] try: # a bit clunky: access to .cn can fail, see https://github.com/mitmproxy/mitmproxy/issues/4713 From aca3456fee699c88647619e938395a8bd2aa4371 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 3 Aug 2021 16:31:57 +0200 Subject: [PATCH 2/5] docs: update ignore domains tutorial --- docs/src/content/howto-ignoredomains.md | 50 ++++++++++--------------- mitmproxy/proxy/server.py | 2 +- 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/docs/src/content/howto-ignoredomains.md b/docs/src/content/howto-ignoredomains.md index c953a2e32..bcdff4f07 100644 --- a/docs/src/content/howto-ignoredomains.md +++ b/docs/src/content/howto-ignoredomains.md @@ -44,7 +44,7 @@ There are two important quirks to consider: information before the SSL handshake. If the client uses SNI however, then we treat the SNI host as an ignore target. - **In regular and upstream proxy mode, explicit HTTP requests are never - ignored.**\[1\] The ignore pattern is applied on CONNECT requests, which + ignored.**[^1] The ignore pattern is applied on CONNECT requests, which initiate HTTPS or clear-text WebSocket connections. ## Tutorial @@ -52,21 +52,23 @@ There are two important quirks to consider: If you just want to ignore one specific domain, there's usually a bulletproof method to do so: -1. Run mitmproxy or mitmdump in verbose mode (`-v`) and observe the `host:port` - information in the serverconnect messages. mitmproxy will filter on these. +1. Run mitmproxy or mitmdump and observe the `host:port` + information following the `server connect` messages in the event log. + mitmproxy will filter on these. 2. Take the `host:port` string, surround it with ^ and $, escape all dots (. becomes \\.) and use this as your ignore pattern: ``` ->>> mitmdump -v -127.0.0.1:50588: clientconnect -127.0.0.1:50588: request - -> CONNECT example.com:443 HTTP/1.1 -127.0.0.1:50588: Set new server address: example.com:443 -127.0.0.1:50588: serverconnect - -> example.com:443 +>>> mitmdump +Proxy server listening at http://*:8080 +127.0.0.1:57089: client connect +127.0.0.1:57089: server connect example.com:443 (93.184.216.34:443) +127.0.0.1:57089: GET https://example.com/ HTTP/2.0 + << HTTP/2.0 200 OK 1.23k +127.0.0.1:57089: client disconnect +127.0.0.1:57089: server disconnect example.com:443 (93.184.216.34:443) ^C ->>> mitmproxy --ignore-hosts ^example\.com:443$ +>>> mitmproxy --ignore-hosts '^example\.com:443$' ``` Here are some other examples for ignore patterns: @@ -86,23 +88,11 @@ Here are some other examples for ignore patterns: --ignore-hosts 17\.178\.\d+\.\d+:443 ``` -This option can also be used to only allow some specific domains through negative lookahead expressions. However, ignore -patterns are always matched against the IP address of the target before being matched against its domain name. Thus, the -pattern must allow any IP addresses using an expression like `^(?![0-9\.]+:)` in order for this to work. -Here are examples of such patterns: +If you want to capture some specific domains only, you can use the `--allow-hosts` option, which makes mitmproxy +ignore all other traffic. -``` -# Ignore everything but example.com and mitmproxy.org (not subdomains): ---ignore-hosts '^(?![0-9\.]+:)(?!example\.com:)(?!mitmproxy\.org:)' - -# Ignore everything but example.com and its subdomains: ---ignore-hosts '^(?![0-9\.]+:)(?!([^\.:]+\.)*example\.com:)' -``` - -**Footnotes** - -1. This stems from an limitation of explicit HTTP proxying: A single connection - can be re-used for multiple target domains - a `GET http://example.com/` - request may be followed by a `GET http://evil.com/` request on the same - connection. If we start to ignore the connection after the first request, we - would miss the relevant second one. +[^1]: This stems from an limitation of explicit HTTP proxying: A single connection + can be re-used for multiple target domains - a `GET http://example.com/` + request may be followed by a `GET http://evil.com/` request on the same + connection. If we start to ignore the connection after the first request, we + would miss the relevant second one. diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 18a421e87..cb434f650 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -173,7 +173,7 @@ class ConnectionHandler(metaclass=abc.ABCMeta): assert command.connection.peername if command.connection.address[0] != command.connection.peername[0]: - addr = f"{command.connection.address[0]} ({human.format_address(command.connection.peername)})" + addr = f"{human.format_address(command.connection.address)} ({human.format_address(command.connection.peername)})" else: addr = human.format_address(command.connection.address) self.log(f"server connect {addr}") From 359406e7a5e9b4f9c51ef58854cea4349e99b364 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 3 Aug 2021 16:32:38 +0200 Subject: [PATCH 3/5] update certificate docs --- docs/src/content/concepts-certificates.md | 86 ++++++++++++++--------- docs/src/content/overview-features.md | 1 - 2 files changed, 52 insertions(+), 35 deletions(-) diff --git a/docs/src/content/concepts-certificates.md b/docs/src/content/concepts-certificates.md index acf57a705..78f5e77f7 100644 --- a/docs/src/content/concepts-certificates.md +++ b/docs/src/content/concepts-certificates.md @@ -8,13 +8,13 @@ menu: # About Certificates Mitmproxy can decrypt encrypted traffic on the fly, as long as the client trusts -its built-in certificate authority. Usually this means that the mitmproxy CA -certificates have to be installed on the client device. +mitmproxy's built-in certificate authority. Usually this means that the mitmproxy CA +certificate has to be installed on the client device. ## Quick Setup -By far the easiest way to install the mitmproxy certificates is to use the -built-in certificate installation app. To do this, just start mitmproxy and +By far the easiest way to install the mitmproxy CA certificate is to use the +built-in certificate installation app. To do this, start mitmproxy and configure your target device with the correct proxy settings. Now start a browser on the device, and visit the magic domain [mitm.it](http://mitm.it/). You should see something like this: @@ -24,11 +24,33 @@ something like this: Click on the relevant icon, follow the setup instructions for the platform you're on and you are good to go. -## Installing the mitmproxy CA certificate manually +## The mitmproxy certificate authority -Sometimes using the quick install app is not an option - Java or the iOS -Simulator spring to mind - or you just need to do it manually for some other -reason. Below is a list of pointers to manual certificate installation +The first time mitmproxy is run, it creates the keys for a certificate +authority (CA) in the config directory (`~/.mitmproxy` by default). +This CA is used for on-the-fly generation of dummy certificates for each visited website. +Since your browser won't trust the mitmproxy CA out of the box, you will either need to click through a TLS certificate +warning on every domain, or install the CA certificate once so that it is trusted. + +The following files are created: + +| Filename | Contents | +| --------------------- | ------------------------------------------------------------------------------------ | +| mitmproxy-ca.pem | The certificate **and the private key** in PEM format. | +| mitmproxy-ca-cert.pem | The certificate in PEM format. Use this to distribute on most non-Windows platforms. | +| mitmproxy-ca-cert.p12 | The certificate in PKCS12 format. For use on Windows. | +| mitmproxy-ca-cert.cer | Same file as .pem, but with an extension expected by some Android devices. | + +For security reasons, the mitmproxy CA is generated uniquely on the first start and +is not shared between mitmproxy installations on different devices. This makes sure +that other mitmproxy users cannot intercept your traffic. + + + +### Installing the mitmproxy CA certificate manually + +Sometimes using the [quick install app](#quick-setup) is not an option and you need to install the CA manually. +Below is a list of pointers to manual certificate installation documentation for some common platforms. The mitmproxy CA cert is located in `~/.mitmproxy` after it has been generated at the first start of mitmproxy. @@ -56,42 +78,38 @@ documentation for some common platforms. The mitmproxy CA cert is located in - [Windows (automated)](https://technet.microsoft.com/en-us/library/cc732443.aspx): `certutil -addstore root mitmproxy-ca-cert.cer` -## The mitmproxy certificate authority +### Upstream Certificate Sniffing + +When mitmproxy receives a request to establish TLS (in the form of a ClientHello message), it puts the client on hold +and first makes a connection to the upstream server to "sniff" the contents of its TLS certificate. +The information gained -- Common Name, Organization, Subject Alternative Names -- is then used to generate a new +interception certificate on-the-fly, signed by the mitmproxy CA. Mitmproxy then returns to the client and continues +the handshake with the newly-forged certificate. + +Upstream cert sniffing is on by default, and can optionally be disabled by turning the `upstream_cert` option off. -The first time **mitmproxy** or **mitmdump** is run, the mitmproxy Certificate -Authority (CA) is created in the config directory (`~/.mitmproxy` by default). -This CA is used for on-the-fly generation of dummy certificates for each of the -SSL sites that your client visits. Since your browser won't trust the mitmproxy -CA out of the box, you will see an SSL certificate warning every time you visit -a new SSL domain through mitmproxy. When you are testing a single site through a -browser, just accepting the bogus SSL cert manually is not too much trouble, but -there are many circumstances where you will want to configure your testing -system or browser to trust the mitmproxy CA as a signing root authority. For -security reasons, the mitmproxy CA is generated uniquely on the first start and -is not shared between mitmproxy installations on different devices. ### Certificate Pinning Some applications employ [Certificate Pinning](https://en.wikipedia.org/wiki/HTTP_Public_Key_Pinning) to prevent -man-in-the-middle attacks. This means that **mitmproxy** and **mitmdump's** +man-in-the-middle attacks. This means that **mitmproxy's** certificates will not be accepted by these applications without modifying them. -It is recommended to use the passthrough feature in order to prevent -**mitmproxy** and **mitmdump** from intercepting traffic to these specific +If the contents of these connections are not important, it is recommended to use +the [ignore_hosts]({{< relref "howto-ignoredomains">}}) feature to prevent +**mitmproxy** from intercepting traffic to these specific domains. If you want to intercept the pinned connections, you need to patch the application manually. For Android and (jailbroken) iOS devices, various tools -exist to accomplish this. +exist to accomplish this: -## CA and cert files + - [apk-mitm](https://github.com/shroudedcode/apk-mitm) is a CLI application that automatically removes certificate + pinning from Android APK files. + - [objection](https://github.com/sensepost/objection) is a runtime mobile exploration toolkit powered by Frida, + which supports certificate pinning bypasses on iOS and Android. + - [ssl-kill-switch2](https://github.com/nabla-c0d3/ssl-kill-switch2) is a blackbox tool to disable certificate pinning + within iOS and macOS applications. -The files created by mitmproxy in the .mitmproxy directory are as follows: - -| | | -| --------------------- | ------------------------------------------------------------------------------------ | -| mitmproxy-ca.pem | The certificate **and the private key** in PEM format. | -| mitmproxy-ca-cert.pem | The certificate in PEM format. Use this to distribute on most non-Windows platforms. | -| mitmproxy-ca-cert.p12 | The certificate in PKCS12 format. For use on Windows. | -| mitmproxy-ca-cert.cer | Same file as .pem, but with an extension expected by some Android devices. | +*Please propose other useful tools using the "Edit on GitHub" button on the top right of this page.* ## Using a custom server certificate @@ -177,7 +195,7 @@ use it to generate certificates: You can use a client certificate by passing the `--set client_certs=DIRECTORY|FILE` option to mitmproxy. Using a directory allows certs to be selected based on hostname, while using a filename allows a single specific certificate to be used -for all SSL connections. Certificate files must be in the PEM format and should +for all TLS connections. Certificate files must be in the PEM format and should contain both the unencrypted private key and the certificate. ### Multiple client certificates diff --git a/docs/src/content/overview-features.md b/docs/src/content/overview-features.md index cce0a5574..618d02838 100644 --- a/docs/src/content/overview-features.md +++ b/docs/src/content/overview-features.md @@ -19,7 +19,6 @@ menu: - [Sticky Auth](#sticky-auth) - [Sticky Cookies](#sticky-cookies) - [Streaming](#streaming) -- [Upstream Certificates](#upstream-certificates) ## Anticache From 416e5046bc2d80ca78ba8446d44d18fb60713ebe Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 3 Aug 2021 16:59:42 +0200 Subject: [PATCH 4/5] add proxy-auth header for plain http requests, fix #4728 --- mitmproxy/addons/upstream_auth.py | 2 +- mitmproxy/test/tflow.py | 6 +++--- test/mitmproxy/addons/test_upstream_auth.py | 14 ++++++-------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/mitmproxy/addons/upstream_auth.py b/mitmproxy/addons/upstream_auth.py index 96c2767f7..c3053bed1 100644 --- a/mitmproxy/addons/upstream_auth.py +++ b/mitmproxy/addons/upstream_auth.py @@ -51,7 +51,7 @@ class UpstreamAuth: def requestheaders(self, f: http.HTTPFlow): if self.auth: - if f.mode == "upstream" and not f.server_conn.via: + if ctx.options.mode.startswith("upstream") and f.request.scheme == "http": f.request.headers["Proxy-Authorization"] = self.auth elif ctx.options.mode.startswith("reverse"): f.request.headers["Authorization"] = self.auth diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py index 615e92434..fc477be9d 100644 --- a/mitmproxy/test/tflow.py +++ b/mitmproxy/test/tflow.py @@ -10,7 +10,7 @@ from mitmproxy.test.tutils import treq, tresp from wsproto.frame_protocol import Opcode -def ttcpflow(client_conn=True, server_conn=True, messages=True, err=None): +def ttcpflow(client_conn=True, server_conn=True, messages=True, err=None) -> tcp.TCPFlow: if client_conn is True: client_conn = tclient_conn() if server_conn is True: @@ -91,7 +91,7 @@ def twebsocketflow(messages=True, err=None, close_code=None, close_reason='') -> return flow -def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None): +def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None) -> http.HTTPFlow: """ @type client_conn: bool | None | mitmproxy.proxy.connection.ClientConnection @type server_conn: bool | None | mitmproxy.proxy.connection.ServerConnection @@ -126,7 +126,7 @@ class DummyFlow(flow.Flow): super().__init__("dummy", client_conn, server_conn, live) -def tdummyflow(client_conn=True, server_conn=True, err=None): +def tdummyflow(client_conn=True, server_conn=True, err=None) -> DummyFlow: if client_conn is True: client_conn = tclient_conn() if server_conn is True: diff --git a/test/mitmproxy/addons/test_upstream_auth.py b/test/mitmproxy/addons/test_upstream_auth.py index 8c8627275..67ec55927 100644 --- a/test/mitmproxy/addons/test_upstream_auth.py +++ b/test/mitmproxy/addons/test_upstream_auth.py @@ -33,20 +33,18 @@ def test_simple(): tctx.configure(up, upstream_auth="foo:bar") f = tflow.tflow() - f.mode = "upstream" - up.requestheaders(f) + up.http_connect_upstream(f) assert "proxy-authorization" in f.request.headers f = tflow.tflow() up.requestheaders(f) assert "proxy-authorization" not in f.request.headers + assert "authorization" not in f.request.headers + + tctx.configure(up, mode="upstream:127.0.0.1") + up.requestheaders(f) + assert "proxy-authorization" in f.request.headers tctx.configure(up, mode="reverse:127.0.0.1") - f = tflow.tflow() - f.mode = "transparent" up.requestheaders(f) assert "authorization" in f.request.headers - - f = tflow.tflow() - up.http_connect_upstream(f) - assert "proxy-authorization" in f.request.headers From 45123cd287f8764f42424de8f059fc15483a301e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 3 Aug 2021 17:14:07 +0200 Subject: [PATCH 5/5] update CHANGELOG --- CHANGELOG.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 169766697..565f3dc04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,20 @@ # Release History -## Unreleased: mitmproxy next +## 3 August 2021: mitmproxy 7.0.1 -* Use local IP address as certificate subject if no other info is available (@mhils). -* Disable HTTP/2 CONNECT for Secure Web Proxies to fix compatibility with Firefox. (@mhils) -* Allow no-op assignments to `Server.address` when connection is open. (@SaladDais) -* Performance: Re-use OpenSSL context to enable TLS session resumption. (@mhils) +* Performance: Re-use OpenSSL contexts to enable TLS session resumption (@mhils) +* Disable HTTP/2 CONNECT for Secure Web Proxies to fix compatibility with Firefox (@mhils) +* Use local IP address as certificate subject if no other info is available (@mhils) +* Make it possible to return multiple chunks for HTTP stream modification (@mhils) +* Don't send WebSocket CONTINUATION frames when the peer does not send any (@Pilphe) +* Fix HTTP stream modify example. (@mhils) +* Fix a crash caused by no-op assignments to `Server.address` (@SaladDais) +* Fix a crash when encountering invalid certificates (@mhils) +* Fix a crash when pressing the Home/End keys in some screens (@rbdixon) +* Fix a crash when reading corrupted flow dumps (@mhils) +* Fix multiple crashes on flow export (@mhils) +* Fix a bug where ASGI apps did not see the request body (@mhils) +* Minor documentation improvements (@mhils) ## 16 July 2021: mitmproxy 7.0