From af777f9b4f1f881b9dbdcd8c75e381db3d55ff9e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 15 Jul 2017 12:21:15 +0200 Subject: [PATCH 1/5] remove `--shortversion` this has no benefit over --version and clutters our cmdline options. --- mitmproxy/tools/cmdline.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 97b04e743..cd62aa2c8 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -12,14 +12,9 @@ def common_options(parser, opts): parser.add_argument( '--version', action='store_true', + help="show version number and exit", dest='version', ) - parser.add_argument( - '--shortversion', - action='version', - help="show program's short version number and exit", - version=version.VERSION - ) parser.add_argument( '--options', action='store_true', From a81cd18d92be3ec6e326fc039a5282dd927ccc3a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 15 Jul 2017 12:22:46 +0200 Subject: [PATCH 2/5] make eventsequence.iterate extendable --- mitmproxy/eventsequence.py | 94 ++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/mitmproxy/eventsequence.py b/mitmproxy/eventsequence.py index 4e1999726..c2455f245 100644 --- a/mitmproxy/eventsequence.py +++ b/mitmproxy/eventsequence.py @@ -1,4 +1,6 @@ -from mitmproxy import controller +import typing + +from mitmproxy import controller, flow from mitmproxy import http from mitmproxy import tcp from mitmproxy import websocket @@ -8,27 +10,22 @@ Events = frozenset([ "clientdisconnect", "serverconnect", "serverdisconnect", - "tcp_start", "tcp_message", "tcp_error", "tcp_end", - "http_connect", "request", "requestheaders", "response", "responseheaders", "error", - "websocket_handshake", "websocket_start", "websocket_message", "websocket_error", "websocket_end", - "next_layer", - "configure", "done", "log", @@ -39,37 +36,56 @@ Events = frozenset([ ]) -def iterate(f): - if isinstance(f, http.HTTPFlow): - if f.request: - yield "requestheaders", f - yield "request", f - if f.response: - yield "responseheaders", f - yield "response", f - if f.error: - yield "error", f - elif isinstance(f, websocket.WebSocketFlow): - messages = f.messages - f.messages = [] - f.reply = controller.DummyReply() - yield "websocket_start", f - while messages: - f.messages.append(messages.pop(0)) - yield "websocket_message", f - if f.error: - yield "websocket_error", f - yield "websocket_end", f - elif isinstance(f, tcp.TCPFlow): - messages = f.messages - f.messages = [] - f.reply = controller.DummyReply() - yield "tcp_start", f - while messages: - f.messages.append(messages.pop(0)) - yield "tcp_message", f - if f.error: - yield "tcp_error", f - yield "tcp_end", f +def _iterate_http(f: http.HTTPFlow): + if f.request: + yield "requestheaders", f + yield "request", f + if f.response: + yield "responseheaders", f + yield "response", f + if f.error: + yield "error", f + + +def _iterate_websocket(f: websocket.WebSocketFlow): + messages = f.messages + f.messages = [] + f.reply = controller.DummyReply() + yield "websocket_start", f + while messages: + f.messages.append(messages.pop(0)) + yield "websocket_message", f + if f.error: + yield "websocket_error", f + yield "websocket_end", f + + +def _iterate_tcp(f: tcp.TCPFlow): + messages = f.messages + f.messages = [] + f.reply = controller.DummyReply() + yield "tcp_start", f + while messages: + f.messages.append(messages.pop(0)) + yield "tcp_message", f + if f.error: + yield "tcp_error", f + yield "tcp_end", f + + +TEventGenerator = typing.Iterator[typing.Tuple[str, typing.Any]] + +_iterate_map: typing.Dict[flow.Flow, typing.Callable[[flow.Flow], TEventGenerator]] = { + http.HTTPFlow: _iterate_http, + websocket.WebSocketFlow: _iterate_websocket, + tcp.TCPFlow: _iterate_tcp +} + + +def iterate(f: flow.Flow) -> TEventGenerator: + try: + e = _iterate_map[type(f)] + except KeyError as e: + raise TypeError("Unknown flow type: {}".format(f)) from e else: - raise TypeError() + yield from e(f) From b7bda8f4b216fb278fb688cc656e4e6e5f6423a0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 17 Jul 2017 20:47:44 +0200 Subject: [PATCH 3/5] shorten option help We can have longer versions in the docs, but this makes the options pages quite hard to parse. --- mitmproxy/eventsequence.py | 11 ++++++++--- mitmproxy/options.py | 23 +++++++++-------------- mitmproxy/tools/cmdline.py | 1 - 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/mitmproxy/eventsequence.py b/mitmproxy/eventsequence.py index c2455f245..09322a8fd 100644 --- a/mitmproxy/eventsequence.py +++ b/mitmproxy/eventsequence.py @@ -1,7 +1,8 @@ import typing -from mitmproxy import controller, flow +from mitmproxy import controller from mitmproxy import http +from mitmproxy import flow from mitmproxy import tcp from mitmproxy import websocket @@ -10,21 +11,25 @@ Events = frozenset([ "clientdisconnect", "serverconnect", "serverdisconnect", + # TCP "tcp_start", "tcp_message", "tcp_error", "tcp_end", + # HTTP "http_connect", "request", "requestheaders", "response", "responseheaders", "error", + # WebSocket "websocket_handshake", "websocket_start", "websocket_message", "websocket_error", "websocket_end", + # misc "next_layer", "configure", "done", @@ -75,11 +80,11 @@ def _iterate_tcp(f: tcp.TCPFlow): TEventGenerator = typing.Iterator[typing.Tuple[str, typing.Any]] -_iterate_map: typing.Dict[flow.Flow, typing.Callable[[flow.Flow], TEventGenerator]] = { +_iterate_map = { http.HTTPFlow: _iterate_http, websocket.WebSocketFlow: _iterate_websocket, tcp.TCPFlow: _iterate_tcp -} +} # type: typing.Dict[flow.Flow, typing.Callable[[flow.Flow], TEventGenerator]] def iterate(f: flow.Flow) -> TEventGenerator: diff --git a/mitmproxy/options.py b/mitmproxy/options.py index e6c2fed67..954db7e8c 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -209,15 +209,11 @@ class Options(optmanager.OptManager): self.add_option( "proxyauth", Optional[str], None, """ - Require proxy authentication. Value may be "any" to require - authenticaiton but accept any credentials, start with "@" to specify - a path to an Apache htpasswd file, be of the form - "username:password", or be of the form - "ldap[s]:url_server_ldap:dn_auth:password:dn_subtree", - the dn_auth & password is the dn/pass used to authenticate - the dn subtree is the subtree that we will search to find the username - an example would be - "ldap:localhost:cn=default,dc=example,dc=com:password:ou=application,dc=example,dc=com". + Require proxy authentication. Format: + "username:pass", + "any" to accept any user/pass combination, + "@path" to use an Apache htpasswd file, + or "ldap[s]:url_server_ldap:dn_auth:password:dn_subtree" for LDAP authentication. """ ) self.add_option( @@ -288,7 +284,7 @@ class Options(optmanager.OptManager): """ Mode can be "regular", "transparent", "socks5", "reverse:SPEC", or "upstream:SPEC". For reverse and upstream proxy modes, SPEC - is proxy specification in the form of "http[s]://host[:port]". + is host specification in the form of "http[s]://host[:port]". """ ) self.add_option( @@ -311,9 +307,8 @@ class Options(optmanager.OptManager): self.add_option( "http2_priority", bool, False, """ - PRIORITY forwarding for HTTP/2 connections. PRIORITY forwarding is - disabled by default, because some webservers fail to implement the - RFC properly. + PRIORITY forwarding for HTTP/2 connections. Disabled by default to ensure compatibility + with misbehaving servers. """ ) self.add_option( @@ -337,7 +332,7 @@ class Options(optmanager.OptManager): self.add_option( "upstream_auth", Optional[str], None, """ - Add HTTP Basic authentcation to upstream proxy and reverse proxy + Add HTTP Basic authentication to upstream proxy and reverse proxy requests. Format: username:password. """ ) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index cd62aa2c8..68ddc2c8e 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -2,7 +2,6 @@ import argparse import os from mitmproxy import options -from mitmproxy import version CONFIG_PATH = os.path.join(options.CA_DIR, "config.yaml") From 9176626dab56f9437324b56ac0b2751834dc3ead Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 20 Jul 2017 15:43:36 +0200 Subject: [PATCH 4/5] fix type declaration --- mitmproxy/eventsequence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/eventsequence.py b/mitmproxy/eventsequence.py index 09322a8fd..68d97a23b 100644 --- a/mitmproxy/eventsequence.py +++ b/mitmproxy/eventsequence.py @@ -84,7 +84,7 @@ _iterate_map = { http.HTTPFlow: _iterate_http, websocket.WebSocketFlow: _iterate_websocket, tcp.TCPFlow: _iterate_tcp -} # type: typing.Dict[flow.Flow, typing.Callable[[flow.Flow], TEventGenerator]] +} # type: typing.Dict[typing.Type[flow.Flow], typing.Callable[[flow.Flow], TEventGenerator]] def iterate(f: flow.Flow) -> TEventGenerator: From 7b0485d6d6d1f3cc5d6704db39971b69dc030b80 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 20 Jul 2017 16:27:13 +0200 Subject: [PATCH 5/5] update mypy --- mitmproxy/addons/core.py | 4 ++-- mitmproxy/eventsequence.py | 20 ++++++++++---------- mitmproxy/utils/human.py | 2 +- setup.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index b0d8ee275..33d672790 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -112,7 +112,7 @@ class Core: val = sval # type: typing.Union[int, str] if spec == "status_code": try: - val = int(val) + val = int(val) # type: ignore except ValueError as v: raise exceptions.CommandError( "Status code is not an integer: %s" % val @@ -145,7 +145,7 @@ class Core: if spec == "status_code": resp.status_code = val if val in status_codes.RESPONSES: - resp.reason = status_codes.RESPONSES[int(val)] + resp.reason = status_codes.RESPONSES[val] # type: ignore elif spec == "reason": resp.reason = val else: diff --git a/mitmproxy/eventsequence.py b/mitmproxy/eventsequence.py index 68d97a23b..d263727b7 100644 --- a/mitmproxy/eventsequence.py +++ b/mitmproxy/eventsequence.py @@ -1,8 +1,8 @@ import typing from mitmproxy import controller -from mitmproxy import http from mitmproxy import flow +from mitmproxy import http from mitmproxy import tcp from mitmproxy import websocket @@ -40,8 +40,10 @@ Events = frozenset([ "update", ]) +TEventGenerator = typing.Iterator[typing.Tuple[str, typing.Any]] -def _iterate_http(f: http.HTTPFlow): + +def _iterate_http(f: http.HTTPFlow) -> TEventGenerator: if f.request: yield "requestheaders", f yield "request", f @@ -52,7 +54,7 @@ def _iterate_http(f: http.HTTPFlow): yield "error", f -def _iterate_websocket(f: websocket.WebSocketFlow): +def _iterate_websocket(f: websocket.WebSocketFlow) -> TEventGenerator: messages = f.messages f.messages = [] f.reply = controller.DummyReply() @@ -65,7 +67,7 @@ def _iterate_websocket(f: websocket.WebSocketFlow): yield "websocket_end", f -def _iterate_tcp(f: tcp.TCPFlow): +def _iterate_tcp(f: tcp.TCPFlow) -> TEventGenerator: messages = f.messages f.messages = [] f.reply = controller.DummyReply() @@ -78,19 +80,17 @@ def _iterate_tcp(f: tcp.TCPFlow): yield "tcp_end", f -TEventGenerator = typing.Iterator[typing.Tuple[str, typing.Any]] - _iterate_map = { http.HTTPFlow: _iterate_http, websocket.WebSocketFlow: _iterate_websocket, - tcp.TCPFlow: _iterate_tcp -} # type: typing.Dict[typing.Type[flow.Flow], typing.Callable[[flow.Flow], TEventGenerator]] + tcp.TCPFlow: _iterate_tcp, +} # type: typing.Dict[typing.Type[flow.Flow], typing.Callable[[typing.Any], TEventGenerator]] def iterate(f: flow.Flow) -> TEventGenerator: try: e = _iterate_map[type(f)] - except KeyError as e: - raise TypeError("Unknown flow type: {}".format(f)) from e + except KeyError as err: + raise TypeError("Unknown flow type: {}".format(f)) from err else: yield from e(f) diff --git a/mitmproxy/utils/human.py b/mitmproxy/utils/human.py index b3934846c..d67fb3104 100644 --- a/mitmproxy/utils/human.py +++ b/mitmproxy/utils/human.py @@ -71,7 +71,7 @@ def format_address(address: tuple) -> str: """ try: host = ipaddress.ip_address(address[0]) - if host.version == 4: + if isinstance(host, ipaddress.IPv4Address): return "{}:{}".format(str(host), address[1]) # If IPv6 is mapped to IPv4 elif host.ipv4_mapped: diff --git a/setup.py b/setup.py index 212ad95ea..441a1d8f9 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ setup( 'dev': [ "flake8>=3.2.1, <3.4", "Flask>=0.10.1, <0.13", - "mypy>=0.501, <0.512", + "mypy>=0.501, <0.521", "pytest-cov>=2.2.1, <3", "pytest-faulthandler>=1.3.0, <2", "pytest-timeout>=1.0.0, <2",