Merge pull request #1967 from s4chin/remove-pillow

Add png parser

mitmproxy/contentviews/image/__init__.py

@@ -0,0 +1 @@
+from .view import ViewImage  # noqa

mitmproxy/contentviews/image/image_parser.py

@@ -0,0 +1,30 @@
+import io
+import typing
+
+from kaitaistruct import KaitaiStream
+
+from mitmproxy.contrib.kaitaistruct import png
+
+Metadata = typing.List[typing.Tuple[str, str]]
+
+
+def parse_png(data: bytes) -> Metadata:
+    img = png.Png(KaitaiStream(io.BytesIO(data)))
+    parts = [
+        ('Format', 'Portable network graphics')
+    ]
+    parts.append(('Size', "{0} x {1} px".format(img.ihdr.width, img.ihdr.height)))
+    for chunk in img.chunks:
+        if chunk.type == 'gAMA':
+            parts.append(('gamma', str(chunk.body.gamma_int / 100000)))
+        elif chunk.type == 'pHYs':
+            aspectx = chunk.body.pixels_per_unit_x
+            aspecty = chunk.body.pixels_per_unit_y
+            parts.append(('aspect', "{0} x {1}".format(aspectx, aspecty)))
+        elif chunk.type == 'tEXt':
+            parts.append((chunk.body.keyword, chunk.body.text))
+        elif chunk.type == 'iTXt':
+            parts.append((chunk.body.keyword, chunk.body.text))
+        elif chunk.type == 'zTXt':
+            parts.append((chunk.body.keyword, chunk.body.text_datastream.decode('iso8859-1')))
+    return parts

mitmproxy/contentviews/image/view.py

@@ -1,10 +1,13 @@
 import io
+import imghdr
 
 from PIL import ExifTags
 from PIL import Image
 
 from mitmproxy.types import multidict
-from . import base
+from . import image_parser
+
+from mitmproxy.contentviews import base
 
 
 class ViewImage(base.View):
@@ -19,6 +22,11 @@ class ViewImage(base.View):
     ]
 
     def __call__(self, data, **metadata):
+        if imghdr.what('', h=data) == 'png':
+            f = "PNG"
+            parts = image_parser.parse_png(data)
+            fmt = base.format_dict(multidict.MultiDict(parts))
+            return "%s image" % f, fmt
         try:
             img = Image.open(io.BytesIO(data))
         except IOError:

mitmproxy/contrib/kaitaistruct/png.py

@@ -0,0 +1,289 @@
+# This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
+# The source was png.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/9370c720b7d2ad329102d89bdc880ba6a706ef26/image/png.ksy
+
+import array
+import struct
+import zlib
+from enum import Enum
+
+from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO
+
+
+class Png(KaitaiStruct):
+
+    class ColorType(Enum):
+        greyscale = 0
+        truecolor = 2
+        indexed = 3
+        greyscale_alpha = 4
+        truecolor_alpha = 6
+
+    class PhysUnit(Enum):
+        unknown = 0
+        meter = 1
+    def __init__(self, _io, _parent=None, _root=None):
+        self._io = _io
+        self._parent = _parent
+        self._root = _root if _root else self
+        self.magic = self._io.ensure_fixed_contents(8, struct.pack('8b', -119, 80, 78, 71, 13, 10, 26, 10))
+        self.ihdr_len = self._io.ensure_fixed_contents(4, struct.pack('4b', 0, 0, 0, 13))
+        self.ihdr_type = self._io.ensure_fixed_contents(4, struct.pack('4b', 73, 72, 68, 82))
+        self.ihdr = self._root.IhdrChunk(self._io, self, self._root)
+        self.ihdr_crc = self._io.read_bytes(4)
+        self.chunks = []
+        while not self._io.is_eof():
+            self.chunks.append(self._root.Chunk(self._io, self, self._root))
+
+
+    class Rgb(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.r = self._io.read_u1()
+            self.g = self._io.read_u1()
+            self.b = self._io.read_u1()
+
+
+    class Chunk(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.len = self._io.read_u4be()
+            self.type = self._io.read_str_byte_limit(4, "UTF-8")
+            _on = self.type
+            if _on == u"iTXt":
+                self._raw_body = self._io.read_bytes(self.len)
+                io = KaitaiStream(BytesIO(self._raw_body))
+                self.body = self._root.InternationalTextChunk(io, self, self._root)
+            elif _on == u"gAMA":
+                self._raw_body = self._io.read_bytes(self.len)
+                io = KaitaiStream(BytesIO(self._raw_body))
+                self.body = self._root.GamaChunk(io, self, self._root)
+            elif _on == u"tIME":
+                self._raw_body = self._io.read_bytes(self.len)
+                io = KaitaiStream(BytesIO(self._raw_body))
+                self.body = self._root.TimeChunk(io, self, self._root)
+            elif _on == u"PLTE":
+                self._raw_body = self._io.read_bytes(self.len)
+                io = KaitaiStream(BytesIO(self._raw_body))
+                self.body = self._root.PlteChunk(io, self, self._root)
+            elif _on == u"bKGD":
+                self._raw_body = self._io.read_bytes(self.len)
+                io = KaitaiStream(BytesIO(self._raw_body))
+                self.body = self._root.BkgdChunk(io, self, self._root)
+            elif _on == u"pHYs":
+                self._raw_body = self._io.read_bytes(self.len)
+                io = KaitaiStream(BytesIO(self._raw_body))
+                self.body = self._root.PhysChunk(io, self, self._root)
+            elif _on == u"tEXt":
+                self._raw_body = self._io.read_bytes(self.len)
+                io = KaitaiStream(BytesIO(self._raw_body))
+                self.body = self._root.TextChunk(io, self, self._root)
+            elif _on == u"cHRM":
+                self._raw_body = self._io.read_bytes(self.len)
+                io = KaitaiStream(BytesIO(self._raw_body))
+                self.body = self._root.ChrmChunk(io, self, self._root)
+            elif _on == u"sRGB":
+                self._raw_body = self._io.read_bytes(self.len)
+                io = KaitaiStream(BytesIO(self._raw_body))
+                self.body = self._root.SrgbChunk(io, self, self._root)
+            elif _on == u"zTXt":
+                self._raw_body = self._io.read_bytes(self.len)
+                io = KaitaiStream(BytesIO(self._raw_body))
+                self.body = self._root.CompressedTextChunk(io, self, self._root)
+            else:
+                self.body = self._io.read_bytes(self.len)
+            self.crc = self._io.read_bytes(4)
+
+
+    class BkgdIndexed(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.palette_index = self._io.read_u1()
+
+
+    class Point(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.x_int = self._io.read_u4be()
+            self.y_int = self._io.read_u4be()
+
+        @property
+        def x(self):
+            if hasattr(self, '_m_x'):
+                return self._m_x
+
+            self._m_x = (self.x_int / 100000.0)
+            return self._m_x
+
+        @property
+        def y(self):
+            if hasattr(self, '_m_y'):
+                return self._m_y
+
+            self._m_y = (self.y_int / 100000.0)
+            return self._m_y
+
+
+    class BkgdGreyscale(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.value = self._io.read_u2be()
+
+
+    class ChrmChunk(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.white_point = self._root.Point(self._io, self, self._root)
+            self.red = self._root.Point(self._io, self, self._root)
+            self.green = self._root.Point(self._io, self, self._root)
+            self.blue = self._root.Point(self._io, self, self._root)
+
+
+    class IhdrChunk(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.width = self._io.read_u4be()
+            self.height = self._io.read_u4be()
+            self.bit_depth = self._io.read_u1()
+            self.color_type = self._root.ColorType(self._io.read_u1())
+            self.compression_method = self._io.read_u1()
+            self.filter_method = self._io.read_u1()
+            self.interlace_method = self._io.read_u1()
+
+
+    class PlteChunk(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.entries = []
+            while not self._io.is_eof():
+                self.entries.append(self._root.Rgb(self._io, self, self._root))
+
+
+
+    class SrgbChunk(KaitaiStruct):
+
+        class Intent(Enum):
+            perceptual = 0
+            relative_colorimetric = 1
+            saturation = 2
+            absolute_colorimetric = 3
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.render_intent = self._root.Intent(self._io.read_u1())
+
+
+    class CompressedTextChunk(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.keyword = self._io.read_strz("UTF-8", 0, False, True, True)
+            self.compression_method = self._io.read_u1()
+            self._raw_text_datastream = self._io.read_bytes_full()
+            self.text_datastream = zlib.decompress(self._raw_text_datastream)
+
+
+    class BkgdTruecolor(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.red = self._io.read_u2be()
+            self.green = self._io.read_u2be()
+            self.blue = self._io.read_u2be()
+
+
+    class GamaChunk(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.gamma_int = self._io.read_u4be()
+
+        @property
+        def gamma_ratio(self):
+            if hasattr(self, '_m_gamma_ratio'):
+                return self._m_gamma_ratio
+
+            self._m_gamma_ratio = (100000.0 / self.gamma_int)
+            return self._m_gamma_ratio
+
+
+    class BkgdChunk(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            _on = self._root.ihdr.color_type
+            if _on == self._root.ColorType.greyscale_alpha:
+                self.bkgd = self._root.BkgdGreyscale(self._io, self, self._root)
+            elif _on == self._root.ColorType.indexed:
+                self.bkgd = self._root.BkgdIndexed(self._io, self, self._root)
+            elif _on == self._root.ColorType.greyscale:
+                self.bkgd = self._root.BkgdGreyscale(self._io, self, self._root)
+            elif _on == self._root.ColorType.truecolor_alpha:
+                self.bkgd = self._root.BkgdTruecolor(self._io, self, self._root)
+            elif _on == self._root.ColorType.truecolor:
+                self.bkgd = self._root.BkgdTruecolor(self._io, self, self._root)
+
+
+    class PhysChunk(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.pixels_per_unit_x = self._io.read_u4be()
+            self.pixels_per_unit_y = self._io.read_u4be()
+            self.unit = self._root.PhysUnit(self._io.read_u1())
+
+
+    class InternationalTextChunk(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.keyword = self._io.read_strz("UTF-8", 0, False, True, True)
+            self.compression_flag = self._io.read_u1()
+            self.compression_method = self._io.read_u1()
+            self.language_tag = self._io.read_strz("ASCII", 0, False, True, True)
+            self.translated_keyword = self._io.read_strz("UTF-8", 0, False, True, True)
+            self.text = self._io.read_str_eos("UTF-8")
+
+
+    class TextChunk(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.keyword = self._io.read_strz("iso8859-1", 0, False, True, True)
+            self.text = self._io.read_str_eos("iso8859-1")
+
+
+    class TimeChunk(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self.year = self._io.read_u2be()
+            self.month = self._io.read_u1()
+            self.day = self._io.read_u1()
+            self.hour = self._io.read_u1()
+            self.minute = self._io.read_u1()
+            self.second = self._io.read_u1()

setup.py

@@ -71,6 +71,7 @@ setup(
         "html2text>=2016.1.8, <=2016.9.19",
         "hyperframe>=4.0.1, <5",
         "jsbeautifier>=1.6.3, <1.7",
+        "kaitaistruct>=0.5, <0.6",
         "Pillow>=3.2, <4.1",
         "passlib>=1.6.5, <1.8",
         "pyasn1>=0.1.9, <0.2",

test/mitmproxy/contentviews/test_image_parser.py

@@ -0,0 +1,76 @@
+import pytest
+
+from mitmproxy.contentviews.image import image_parser
+from mitmproxy.test import tutils
+
+
+@pytest.mark.parametrize("filename, metadata", {
+    # no textual data
+    "mitmproxy/data/png_parser/ct0n0g04.png": [
+        ('Format', 'Portable network graphics'),
+        ('Size', '32 x 32 px'),
+        ('gamma', '1.0')
+    ],
+    # with textual data
+    "mitmproxy/data/png_parser/ct1n0g04.png": [
+        ('Format', 'Portable network graphics'),
+        ('Size', '32 x 32 px'),
+        ('gamma', '1.0'),
+        ('Title', 'PngSuite'),
+        ('Author', 'Willem A.J. van Schaik\n(willem@schaik.com)'),
+        ('Copyright', 'Copyright Willem van Schaik, Singapore 1995-96'),
+        ('Description', 'A compilation of a set of images created to test the\n'
+         'various color-types of the PNG format. Included are\nblack&white, color,'
+         ' paletted, with alpha channel, with\ntransparency formats. All bit-depths'
+         ' allowed according\nto the spec are present.'),
+        ('Software', 'Created on a NeXTstation color using "pnmtopng".'),
+        ('Disclaimer', 'Freeware.')
+    ],
+    # with compressed textual data
+    "mitmproxy/data/png_parser/ctzn0g04.png": [
+        ('Format', 'Portable network graphics'),
+        ('Size', '32 x 32 px'),
+        ('gamma', '1.0'),
+        ('Title', 'PngSuite'),
+        ('Author', 'Willem A.J. van Schaik\n(willem@schaik.com)'),
+        ('Copyright', 'Copyright Willem van Schaik, Singapore 1995-96'),
+        ('Description', 'A compilation of a set of images created to test the\n'
+         'various color-types of the PNG format. Included are\nblack&white, color,'
+         ' paletted, with alpha channel, with\ntransparency formats. All bit-depths'
+         ' allowed according\nto the spec are present.'),
+        ('Software', 'Created on a NeXTstation color using "pnmtopng".'),
+        ('Disclaimer', 'Freeware.')
+    ],
+    # UTF-8 international text - english
+    "mitmproxy/data/png_parser/cten0g04.png": [
+        ('Format', 'Portable network graphics'),
+        ('Size', '32 x 32 px'),
+        ('gamma', '1.0'),
+        ('Title', 'PngSuite'),
+        ('Author', 'Willem van Schaik (willem@schaik.com)'),
+        ('Copyright', 'Copyright Willem van Schaik, Canada 2011'),
+        ('Description', 'A compilation of a set of images created to test the '
+         'various color-types of the PNG format. Included are black&white, color,'
+         ' paletted, with alpha channel, with transparency formats. All bit-depths'
+         ' allowed according to the spec are present.'),
+        ('Software', 'Created on a NeXTstation color using "pnmtopng".'),
+        ('Disclaimer', 'Freeware.')
+    ],
+    # check gamma value
+    "mitmproxy/data/png_parser/g07n0g16.png": [
+        ('Format', 'Portable network graphics'),
+        ('Size', '32 x 32 px'),
+        ('gamma', '0.7')
+    ],
+    # check aspect value
+    "mitmproxy/data/png_parser/aspect.png": [
+        ('Format', 'Portable network graphics'),
+        ('Size', '1280 x 798 px'),
+        ('aspect', '72 x 72'),
+        ('date:create', '2012-07-11T14:04:52-07:00'),
+        ('date:modify', '2012-07-11T14:04:52-07:00')
+    ],
+}.items())
+def test_parse_png(filename, metadata):
+    with open(tutils.test_data.path(filename), "rb") as f:
+        assert metadata == image_parser.parse_png(f.read())