From eb5f37a7d19d74c88eed25d4b90f98bec1afa4be Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 26 May 2017 16:14:20 +0200 Subject: [PATCH 1/2] fix mypy annotations --- .gitignore | 1 + examples/simple/custom_contentview.py | 6 +----- examples/simple/io_read_dumpfile.py | 2 -- examples/simple/io_write_dumpfile.py | 10 +++++----- mitmproxy/contentviews/__init__.py | 4 ++-- mitmproxy/contentviews/base.py | 26 +++++++++++++++----------- pathod/language/actions.py | 2 +- pathod/language/base.py | 8 +++++--- pathod/language/http.py | 4 +++- pathod/language/http2.py | 8 +++++--- pathod/language/websockets.py | 24 ++++++++---------------- setup.py | 2 +- 12 files changed, 47 insertions(+), 50 deletions(-) diff --git a/.gitignore b/.gitignore index a37a1f316..f88a29174 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ sslkeylogfile.log .python-version coverage.xml web/coverage/ +.mypy_cache/ diff --git a/examples/simple/custom_contentview.py b/examples/simple/custom_contentview.py index 71f925750..b958bdcea 100644 --- a/examples/simple/custom_contentview.py +++ b/examples/simple/custom_contentview.py @@ -3,10 +3,6 @@ This example shows how one can add a custom contentview to mitmproxy. The content view API is explained in the mitmproxy.contentviews module. """ from mitmproxy import contentviews -import typing - - -CVIEWSWAPCASE = typing.Tuple[str, typing.Iterable[typing.List[typing.Tuple[str, typing.AnyStr]]]] class ViewSwapCase(contentviews.View): @@ -17,7 +13,7 @@ class ViewSwapCase(contentviews.View): prompt = ("swap case text", "z") content_types = ["text/plain"] - def __call__(self, data: typing.AnyStr, **metadata) -> CVIEWSWAPCASE: + def __call__(self, data, **metadata) -> contentviews.TViewResult: return "case-swapped text", contentviews.format_text(data.swapcase()) diff --git a/examples/simple/io_read_dumpfile.py b/examples/simple/io_read_dumpfile.py index ea544cc43..534f357b9 100644 --- a/examples/simple/io_read_dumpfile.py +++ b/examples/simple/io_read_dumpfile.py @@ -1,6 +1,4 @@ #!/usr/bin/env python - -# type: ignore # # Simple script showing how to read a mitmproxy dump file # diff --git a/examples/simple/io_write_dumpfile.py b/examples/simple/io_write_dumpfile.py index cf7c4f527..be6e4121b 100644 --- a/examples/simple/io_write_dumpfile.py +++ b/examples/simple/io_write_dumpfile.py @@ -13,15 +13,15 @@ import typing # noqa class Writer: def __init__(self, path: str) -> None: - if path == "-": - f = sys.stdout # type: typing.IO[typing.Any] - else: - f = open(path, "wb") - self.w = io.FlowWriter(f) + self.f = open(path, "wb") # type: typing.IO[bytes] + self.w = io.FlowWriter(self.f) def response(self, flow: http.HTTPFlow) -> None: if random.choice([True, False]): self.w.add(flow) + def done(self): + self.f.close() + addons = [Writer(sys.argv[1])] diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index f57b27c7f..a1866851a 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -25,7 +25,7 @@ from . import ( auto, raw, hex, json, xml_html, html_outline, wbxml, javascript, css, urlencoded, multipart, image, query, protobuf ) -from .base import View, VIEW_CUTOFF, KEY_MAX, format_text, format_dict +from .base import View, VIEW_CUTOFF, KEY_MAX, format_text, format_dict, TViewResult views = [] # type: List[View] content_types_map = {} # type: Dict[str, List[View]] @@ -178,7 +178,7 @@ add(query.ViewQuery()) add(protobuf.ViewProtobuf()) __all__ = [ - "View", "VIEW_CUTOFF", "KEY_MAX", "format_text", "format_dict", + "View", "VIEW_CUTOFF", "KEY_MAX", "format_text", "format_dict", "TViewResult", "get", "get_by_shortcut", "add", "remove", "get_content_view", "get_message_content_view", ] diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index 0de4f7865..97740eea0 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -1,20 +1,21 @@ # Default view cutoff *in lines* - -from typing import Iterable, AnyStr, List -from typing import Mapping -from typing import Tuple +import typing VIEW_CUTOFF = 512 KEY_MAX = 30 +TTextType = typing.Union[str, bytes] # FIXME: This should be either bytes or str ultimately. +TViewLine = typing.List[typing.Tuple[str, TTextType]] +TViewResult = typing.Tuple[str, typing.Iterator[TViewLine]] + class View: name = None # type: str - prompt = None # type: Tuple[str,str] - content_types = [] # type: List[str] + prompt = None # type: typing.Tuple[str,str] + content_types = [] # type: typing.List[str] - def __call__(self, data: bytes, **metadata): + def __call__(self, data: bytes, **metadata) -> TViewResult: """ Transform raw data into human-readable output. @@ -38,8 +39,8 @@ class View: def format_dict( - d: Mapping[AnyStr, AnyStr] -) -> Iterable[List[Tuple[str, AnyStr]]]: + d: typing.Mapping[TTextType, TTextType] +) -> typing.Iterator[TViewLine]: """ Helper function that transforms the given dictionary into a list of ("key", key ) @@ -49,7 +50,10 @@ def format_dict( max_key_len = max(len(k) for k in d.keys()) max_key_len = min(max_key_len, KEY_MAX) for key, value in d.items(): - key += b":" if isinstance(key, bytes) else u":" + if isinstance(key, bytes): + key += b":" + else: + key += ":" key = key.ljust(max_key_len + 2) yield [ ("header", key), @@ -57,7 +61,7 @@ def format_dict( ] -def format_text(text: AnyStr) -> Iterable[List[Tuple[str, AnyStr]]]: +def format_text(text: TTextType) -> typing.Iterator[TViewLine]: """ Helper function that transforms bytes into the view output format. """ diff --git a/pathod/language/actions.py b/pathod/language/actions.py index fc57a18b0..3e48f40dd 100644 --- a/pathod/language/actions.py +++ b/pathod/language/actions.py @@ -50,7 +50,7 @@ class _Action(base.Token): class PauseAt(_Action): - unique_name = None # type: ignore + unique_name = None def __init__(self, offset, seconds): _Action.__init__(self, offset) diff --git a/pathod/language/base.py b/pathod/language/base.py index c88927481..3b33c5edc 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -6,7 +6,9 @@ import pyparsing as pp from mitmproxy.utils import strutils from mitmproxy.utils import human import typing # noqa -from . import generators, exceptions +from . import generators +from . import exceptions +from . import message # noqa class Settings: @@ -375,7 +377,7 @@ class OptionsOrValue(_Component): class Integer(_Component): - bounds = (None, None) # type: typing.Tuple[typing.Union[int, None], typing.Union[int , None]] + bounds = (None, None) # type: typing.Tuple[typing.Optional[int], typing.Optional[int]] preamble = "" def __init__(self, value): @@ -545,7 +547,7 @@ class NestedMessage(Token): A nested message, as an escaped string with a preamble. """ preamble = "" - nest_type = None # type: ignore + nest_type = None # type: typing.Optional[typing.Type[message.Message]] def __init__(self, value): Token.__init__(self) diff --git a/pathod/language/http.py b/pathod/language/http.py index 5cd717a92..1ea55d75a 100644 --- a/pathod/language/http.py +++ b/pathod/language/http.py @@ -54,7 +54,9 @@ class Method(base.OptionsOrValue): class _HeaderMixin: - unique_name = None # type: ignore + @property + def unique_name(self): + return None def format_header(self, key, value): return [key, b": ", value, b"\r\n"] diff --git a/pathod/language/http2.py b/pathod/language/http2.py index 47d6e3706..dc842e594 100644 --- a/pathod/language/http2.py +++ b/pathod/language/http2.py @@ -1,9 +1,9 @@ import pyparsing as pp + from mitmproxy.net import http from mitmproxy.net.http import user_agents, Headers from . import base, message - """ Normal HTTP requests: ::
: @@ -41,7 +41,9 @@ def get_header(val, headers): class _HeaderMixin: - unique_name = None # type: ignore + @property + def unique_name(self): + return None def values(self, settings): return ( @@ -146,7 +148,7 @@ class Times(base.Integer): class Response(_HTTP2Message): - unique_name = None # type: ignore + unique_name = None comps = ( Header, Body, diff --git a/pathod/language/websockets.py b/pathod/language/websockets.py index b4faf59b2..681679649 100644 --- a/pathod/language/websockets.py +++ b/pathod/language/websockets.py @@ -1,10 +1,12 @@ import random import string +import typing # noqa + +import pyparsing as pp + import mitmproxy.net.websockets from mitmproxy.utils import strutils -import pyparsing as pp from . import base, generators, actions, message -import typing # noqa NESTED_LEADER = b"pathod!" @@ -74,7 +76,7 @@ class Times(base.Integer): preamble = "x" -COMPONENTS = ( +COMPONENTS = [ OpCode, Length, # Bit flags @@ -89,14 +91,13 @@ COMPONENTS = ( KeyNone, Key, Times, - Body, RawBody, -) +] class WebsocketFrame(message.Message): - components = COMPONENTS + components = COMPONENTS # type: typing.List[typing.Type[base._Component]] logattrs = ["body"] # Used for nested frames unique_name = "body" @@ -240,14 +241,5 @@ class NestedFrame(base.NestedMessage): nest_type = WebsocketFrame -COMP = typing.Tuple[ - typing.Type[OpCode], typing.Type[Length], typing.Type[Fin], typing.Type[RSV1], typing.Type[RSV2], typing.Type[RSV3], typing.Type[Mask], - typing.Type[actions.PauseAt], typing.Type[actions.DisconnectAt], typing.Type[actions.InjectAt], typing.Type[KeyNone], typing.Type[Key], - typing.Type[Times], typing.Type[Body], typing.Type[RawBody] -] - - class WebsocketClientFrame(WebsocketFrame): - components = typing.cast(COMP, COMPONENTS + ( - NestedFrame, - )) + components = COMPONENTS + [NestedFrame] diff --git a/setup.py b/setup.py index 59493b50d..c2fb4718a 100644 --- a/setup.py +++ b/setup.py @@ -90,7 +90,7 @@ setup( 'dev': [ "flake8>=3.2.1, <3.4", "Flask>=0.10.1, <0.13", - "mypy>=0.501, <0.502", + "mypy>=0.501, <0.512", "pytest-cov>=2.2.1, <3", "pytest-faulthandler>=1.3.0, <2", "pytest-timeout>=1.0.0, <2", From e7990e0983344179e49a46f160e1482a0a0f6aa7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 26 May 2017 16:22:13 +0200 Subject: [PATCH 2/2] fix recursive import --- pathod/language/base.py | 41 ---------------------------- pathod/language/http.py | 2 +- pathod/language/http2.py | 2 +- pathod/language/message.py | 50 +++++++++++++++++++++++++++++++++-- pathod/language/websockets.py | 2 +- 5 files changed, 51 insertions(+), 46 deletions(-) diff --git a/pathod/language/base.py b/pathod/language/base.py index 3b33c5edc..97871e7e6 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -8,7 +8,6 @@ from mitmproxy.utils import human import typing # noqa from . import generators from . import exceptions -from . import message # noqa class Settings: @@ -539,43 +538,3 @@ class IntField(_Component): def spec(self): return "%s%s" % (self.preamble, self.origvalue) - - -class NestedMessage(Token): - - """ - A nested message, as an escaped string with a preamble. - """ - preamble = "" - nest_type = None # type: typing.Optional[typing.Type[message.Message]] - - def __init__(self, value): - Token.__init__(self) - self.value = value - try: - self.parsed = self.nest_type( - self.nest_type.expr().parseString( - value.val.decode(), - parseAll=True - ) - ) - except pp.ParseException as v: - raise exceptions.ParseException(v.msg, v.line, v.col) - - @classmethod - def expr(cls): - e = pp.Literal(cls.preamble).suppress() - e = e + TokValueLiteral.expr() - return e.setParseAction(lambda x: cls(*x)) - - def values(self, settings): - return [ - self.value.get_generator(settings), - ] - - def spec(self): - return "%s%s" % (self.preamble, self.value.spec()) - - def freeze(self, settings): - f = self.parsed.freeze(settings).spec() - return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f.encode(), escape_single_quotes=True))) diff --git a/pathod/language/http.py b/pathod/language/http.py index 1ea55d75a..5a962145d 100644 --- a/pathod/language/http.py +++ b/pathod/language/http.py @@ -253,7 +253,7 @@ class Response(_HTTPMessage): return ":".join([i.spec() for i in self.tokens]) -class NestedResponse(base.NestedMessage): +class NestedResponse(message.NestedMessage): preamble = "s" nest_type = Response diff --git a/pathod/language/http2.py b/pathod/language/http2.py index dc842e594..5b27d5bf9 100644 --- a/pathod/language/http2.py +++ b/pathod/language/http2.py @@ -205,7 +205,7 @@ class Response(_HTTP2Message): return ":".join([i.spec() for i in self.tokens]) -class NestedResponse(base.NestedMessage): +class NestedResponse(message.NestedMessage): preamble = "s" nest_type = Response diff --git a/pathod/language/message.py b/pathod/language/message.py index 6b4c5021a..5dda654b9 100644 --- a/pathod/language/message.py +++ b/pathod/language/message.py @@ -1,8 +1,11 @@ import abc -from . import actions, exceptions -from mitmproxy.utils import strutils import typing # noqa +import pyparsing as pp + +from mitmproxy.utils import strutils +from . import actions, exceptions, base + LOG_TRUNCATE = 1024 @@ -96,3 +99,46 @@ class Message: def __repr__(self): return self.spec() + + +class NestedMessage(base.Token): + """ + A nested message, as an escaped string with a preamble. + """ + preamble = "" + nest_type = None # type: typing.Optional[typing.Type[Message]] + + def __init__(self, value): + super().__init__() + self.value = value + try: + self.parsed = self.nest_type( + self.nest_type.expr().parseString( + value.val.decode(), + parseAll=True + ) + ) + except pp.ParseException as v: + raise exceptions.ParseException(v.msg, v.line, v.col) + + @classmethod + def expr(cls): + e = pp.Literal(cls.preamble).suppress() + e = e + base.TokValueLiteral.expr() + return e.setParseAction(lambda x: cls(*x)) + + def values(self, settings): + return [ + self.value.get_generator(settings), + ] + + def spec(self): + return "%s%s" % (self.preamble, self.value.spec()) + + def freeze(self, settings): + f = self.parsed.freeze(settings).spec() + return self.__class__( + base.TokValueLiteral( + strutils.bytes_to_escaped_str(f.encode(), escape_single_quotes=True) + ) + ) diff --git a/pathod/language/websockets.py b/pathod/language/websockets.py index 681679649..cc00bcf17 100644 --- a/pathod/language/websockets.py +++ b/pathod/language/websockets.py @@ -236,7 +236,7 @@ class WebsocketFrame(message.Message): return ":".join([i.spec() for i in self.tokens]) -class NestedFrame(base.NestedMessage): +class NestedFrame(message.NestedMessage): preamble = "f" nest_type = WebsocketFrame