From 8a6f8bd461af46d5006a6e3756277af416162262 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 15 Feb 2017 00:34:23 +0100 Subject: [PATCH 1/3] nuke Pillow --- docs/install.rst | 4 ++-- mitmproxy/contentviews/image/view.py | 24 +------------------ setup.py | 2 +- test/mitmproxy/__init__.py | 1 - .../mitmproxy/contentviews/image/test_view.py | 1 - test/mitmproxy/proxy/protocol/test_http2.py | 2 -- 6 files changed, 4 insertions(+), 30 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index b9524897f..c90eaf9fa 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -85,7 +85,7 @@ libraries. This was tested on a fully patched installation of Ubuntu 16.04. .. code:: bash - sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev libtiff5-dev libjpeg8-dev zlib1g-dev libwebp-dev + sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev zlib1g-dev sudo pip3 install mitmproxy # or pip3 install --user mitmproxy On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install @@ -104,7 +104,7 @@ libraries. This was tested on a fully patched installation of Fedora 24. .. code:: bash - sudo dnf install make gcc redhat-rpm-config python3-pip python3-devel libffi-devel openssl-devel libtiff-devel libjpeg-devel zlib-devel libwebp-devel openjpeg2-devel + sudo dnf install make gcc redhat-rpm-config python3-pip python3-devel libffi-devel openssl-devel zlib-devel sudo pip3 install mitmproxy # or pip3 install --user mitmproxy Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``. diff --git a/mitmproxy/contentviews/image/view.py b/mitmproxy/contentviews/image/view.py index 8fdb26e95..d77e1ecb9 100644 --- a/mitmproxy/contentviews/image/view.py +++ b/mitmproxy/contentviews/image/view.py @@ -1,13 +1,9 @@ -import io import imghdr -from PIL import Image - +from mitmproxy.contentviews import base from mitmproxy.types import multidict from . import image_parser -from mitmproxy.contentviews import base - class ViewImage(base.View): name = "Image" @@ -16,8 +12,6 @@ class ViewImage(base.View): "image/png", "image/jpeg", "image/gif", - "image/vnd.microsoft.icon", - "image/x-icon", ] def __call__(self, data, **metadata): @@ -37,19 +31,3 @@ class ViewImage(base.View): parts = image_parser.parse_jpeg(data) fmt = base.format_dict(multidict.MultiDict(parts)) return "%s image" % f, fmt - try: - img = Image.open(io.BytesIO(data)) - except IOError: - return None - parts = [ - ("Format", str(img.format_description)), - ("Size", "%s x %s px" % img.size), - ("Mode", str(img.mode)), - ] - for i in sorted(img.info.keys()): - if i != "exif": - parts.append( - (str(i), str(img.info[i])) - ) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % img.format, fmt diff --git a/setup.py b/setup.py index 17d45b42b..1b3f08a4a 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,6 @@ setup( "hyperframe>=4.0.1, <5", "jsbeautifier>=1.6.3, <1.7", "kaitaistruct>=0.6, <0.7", - "Pillow>=3.2, <4.1", "passlib>=1.6.5, <1.8", "pyasn1>=0.1.9, <0.3", "pyOpenSSL>=16.0, <17.0", @@ -118,6 +117,7 @@ setup( 'examples': [ "beautifulsoup4>=4.4.1, <4.6", "pytz>=2015.07.0, <=2016.10", + "Pillow>=3.2, <4.1", ] } ) diff --git a/test/mitmproxy/__init__.py b/test/mitmproxy/__init__.py index 28dc133fc..6f114e181 100644 --- a/test/mitmproxy/__init__.py +++ b/test/mitmproxy/__init__.py @@ -3,5 +3,4 @@ import logging logging.getLogger("hyper").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("passlib").setLevel(logging.WARNING) -logging.getLogger("PIL").setLevel(logging.WARNING) logging.getLogger("tornado").setLevel(logging.WARNING) diff --git a/test/mitmproxy/contentviews/image/test_view.py b/test/mitmproxy/contentviews/image/test_view.py index ee2f9eaa1..84bcea17f 100644 --- a/test/mitmproxy/contentviews/image/test_view.py +++ b/test/mitmproxy/contentviews/image/test_view.py @@ -9,7 +9,6 @@ def test_view_image(): "mitmproxy/data/image.png", "mitmproxy/data/image.gif", "mitmproxy/data/all.jpeg", - "mitmproxy/data/image.ico" ]: with open(tutils.test_data.path(img), "rb") as f: assert v(f.read()) diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index f5d9259da..cede0b805 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -23,8 +23,6 @@ logging.getLogger("hyper.packages.hpack.hpack").setLevel(logging.WARNING) logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(logging.WARNING) logging.getLogger("passlib.utils.compat").setLevel(logging.WARNING) logging.getLogger("passlib.registry").setLevel(logging.WARNING) -logging.getLogger("PIL.Image").setLevel(logging.WARNING) -logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING) # inspect the log: From 0d9c7ce50c4b87efbd2a11785be467dd19048f41 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 15 Feb 2017 13:44:02 +0100 Subject: [PATCH 2/3] image view: add fallback, catch all images but svgs --- mitmproxy/contentviews/auto.py | 2 ++ mitmproxy/contentviews/image/__init__.py | 4 ++- mitmproxy/contentviews/image/image_parser.py | 16 +++++----- mitmproxy/contentviews/image/view.py | 29 +++++++++++-------- .../contentviews/image/test_image_parser.py | 6 ++-- .../mitmproxy/contentviews/image/test_view.py | 7 +++-- test/mitmproxy/contentviews/test_auto.py | 12 ++++++++ 7 files changed, 50 insertions(+), 26 deletions(-) diff --git a/mitmproxy/contentviews/auto.py b/mitmproxy/contentviews/auto.py index 7b3cbd781..d46a1bd3e 100644 --- a/mitmproxy/contentviews/auto.py +++ b/mitmproxy/contentviews/auto.py @@ -18,6 +18,8 @@ class ViewAuto(base.View): return contentviews.content_types_map[ct][0](data, **metadata) elif strutils.is_xml(data): return contentviews.get("XML/HTML")(data, **metadata) + elif ct.startswith("image/"): + return contentviews.get("Image")(data, **metadata) if metadata.get("query"): return contentviews.get("Query")(data, **metadata) if data and strutils.is_mostly_bin(data): diff --git a/mitmproxy/contentviews/image/__init__.py b/mitmproxy/contentviews/image/__init__.py index 0d0f06e0d..33356bd7a 100644 --- a/mitmproxy/contentviews/image/__init__.py +++ b/mitmproxy/contentviews/image/__init__.py @@ -1 +1,3 @@ -from .view import ViewImage # noqa +from .view import ViewImage + +__all__ = ["ViewImage"] diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py index 1ff3cff7d..062fb38e3 100644 --- a/mitmproxy/contentviews/image/image_parser.py +++ b/mitmproxy/contentviews/image/image_parser.py @@ -13,9 +13,9 @@ Metadata = typing.List[typing.Tuple[str, str]] def parse_png(data: bytes) -> Metadata: img = png.Png(KaitaiStream(io.BytesIO(data))) parts = [ - ('Format', 'Portable network graphics') + ('Format', 'Portable network graphics'), + ('Size', "{0} x {1} px".format(img.ihdr.width, img.ihdr.height)) ] - parts.append(('Size', "{0} x {1} px".format(img.ihdr.width, img.ihdr.height))) for chunk in img.chunks: if chunk.type == 'gAMA': parts.append(('gamma', str(chunk.body.gamma_int / 100000))) @@ -34,13 +34,13 @@ def parse_png(data: bytes) -> Metadata: def parse_gif(data: bytes) -> Metadata: img = gif.Gif(KaitaiStream(io.BytesIO(data))) - parts = [ - ('Format', 'Compuserve GIF') - ] - parts.append(('version', "GIF{0}".format(img.header.version.decode('ASCII')))) descriptor = img.logical_screen_descriptor - parts.append(('Size', "{0} x {1} px".format(descriptor.screen_width, descriptor.screen_height))) - parts.append(('background', str(descriptor.bg_color_index))) + parts = [ + ('Format', 'Compuserve GIF'), + ('Version', "GIF{}".format(img.header.version.decode('ASCII'))), + ('Size', "{} x {} px".format(descriptor.screen_width, descriptor.screen_height)), + ('background', str(descriptor.bg_color_index)) + ] ext_blocks = [] for block in img.blocks: if block.block_type.name == 'extension': diff --git a/mitmproxy/contentviews/image/view.py b/mitmproxy/contentviews/image/view.py index d77e1ecb9..95ee1e436 100644 --- a/mitmproxy/contentviews/image/view.py +++ b/mitmproxy/contentviews/image/view.py @@ -8,26 +8,31 @@ from . import image_parser class ViewImage(base.View): name = "Image" prompt = ("image", "i") + + # there is also a fallback in the auto view for image/*. content_types = [ "image/png", "image/jpeg", "image/gif", + "image/vnd.microsoft.icon", + "image/x-icon", + "image/webp", ] def __call__(self, data, **metadata): image_type = imghdr.what('', h=data) if image_type == 'png': - f = "PNG" - parts = image_parser.parse_png(data) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % f, fmt + image_metadata = image_parser.parse_png(data) elif image_type == 'gif': - f = "GIF" - parts = image_parser.parse_gif(data) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % f, fmt + image_metadata = image_parser.parse_gif(data) elif image_type == 'jpeg': - f = "JPEG" - parts = image_parser.parse_jpeg(data) - fmt = base.format_dict(multidict.MultiDict(parts)) - return "%s image" % f, fmt + image_metadata = image_parser.parse_jpeg(data) + else: + image_metadata = [ + ("Image Format", image_type or "unknown") + ] + if image_type: + view_name = "{} Image".format(image_type.upper()) + else: + view_name = "Unknown Image" + return view_name, base.format_dict(multidict.MultiDict(image_metadata)) diff --git a/test/mitmproxy/contentviews/image/test_image_parser.py b/test/mitmproxy/contentviews/image/test_image_parser.py index 3c8bfdf7f..3cb44ca6a 100644 --- a/test/mitmproxy/contentviews/image/test_image_parser.py +++ b/test/mitmproxy/contentviews/image/test_image_parser.py @@ -80,7 +80,7 @@ def test_parse_png(filename, metadata): # check comment "mitmproxy/data/image_parser/hopper.gif": [ ('Format', 'Compuserve GIF'), - ('version', 'GIF89a'), + ('Version', 'GIF89a'), ('Size', '128 x 128 px'), ('background', '0'), ('comment', "b'File written by Adobe Photoshop\\xa8 4.0'") @@ -88,7 +88,7 @@ def test_parse_png(filename, metadata): # check background "mitmproxy/data/image_parser/chi.gif": [ ('Format', 'Compuserve GIF'), - ('version', 'GIF89a'), + ('Version', 'GIF89a'), ('Size', '320 x 240 px'), ('background', '248'), ('comment', "b'Created with GIMP'") @@ -96,7 +96,7 @@ def test_parse_png(filename, metadata): # check working with color table "mitmproxy/data/image_parser/iss634.gif": [ ('Format', 'Compuserve GIF'), - ('version', 'GIF89a'), + ('Version', 'GIF89a'), ('Size', '245 x 245 px'), ('background', '0') ], diff --git a/test/mitmproxy/contentviews/image/test_view.py b/test/mitmproxy/contentviews/image/test_view.py index 84bcea17f..34f655a13 100644 --- a/test/mitmproxy/contentviews/image/test_view.py +++ b/test/mitmproxy/contentviews/image/test_view.py @@ -9,8 +9,11 @@ def test_view_image(): "mitmproxy/data/image.png", "mitmproxy/data/image.gif", "mitmproxy/data/all.jpeg", + # https://bugs.python.org/issue21574 + # "mitmproxy/data/image.ico", ]: with open(tutils.test_data.path(img), "rb") as f: - assert v(f.read()) + viewname, lines = v(f.read()) + assert img.split(".")[-1].upper() in viewname - assert not v(b"flibble") + assert v(b"flibble") == ('Unknown Image', [[('header', 'Image Format: '), ('text', 'unknown')]]) diff --git a/test/mitmproxy/contentviews/test_auto.py b/test/mitmproxy/contentviews/test_auto.py index a077affa6..2ff43139d 100644 --- a/test/mitmproxy/contentviews/test_auto.py +++ b/test/mitmproxy/contentviews/test_auto.py @@ -30,6 +30,18 @@ def test_view_auto(): ) assert f[0].startswith("XML") + f = v( + b"", + headers=http.Headers(content_type="image/svg+xml") + ) + assert f[0].startswith("XML") + + f = v( + b"verybinary", + headers=http.Headers(content_type="image/new-magic-image-format") + ) + assert f[0] == "Unknown Image" + f = v(b"\xFF" * 30) assert f[0] == "Hex" From 80113a9d6ec63d665f77db16aeb8341c41319bee Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 15 Feb 2017 14:08:36 +0100 Subject: [PATCH 3/3] remove zlib1g dependency from install instructions --- docs/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index c90eaf9fa..cf93cc58e 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -85,7 +85,7 @@ libraries. This was tested on a fully patched installation of Ubuntu 16.04. .. code:: bash - sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev zlib1g-dev + sudo apt-get install python3-dev python3-pip libffi-dev libssl-dev sudo pip3 install mitmproxy # or pip3 install --user mitmproxy On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install @@ -104,7 +104,7 @@ libraries. This was tested on a fully patched installation of Fedora 24. .. code:: bash - sudo dnf install make gcc redhat-rpm-config python3-pip python3-devel libffi-devel openssl-devel zlib-devel + sudo dnf install make gcc redhat-rpm-config python3-devel python3-pip libffi-devel openssl-devel sudo pip3 install mitmproxy # or pip3 install --user mitmproxy Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``.