From dadb50946e3ac489451bda079730e09f84e54d5a Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 16 Apr 2017 15:05:58 +0200 Subject: [PATCH 1/2] update kaitai and recompile formats --- .gitignore | 1 + mitmproxy/contentviews/image/image_parser.py | 2 +- mitmproxy/contrib/kaitaistruct/exif.py | 12 ++++++--- mitmproxy/contrib/kaitaistruct/exif_be.py | 9 +++++-- mitmproxy/contrib/kaitaistruct/exif_le.py | 9 +++++-- mitmproxy/contrib/kaitaistruct/gif.py | 18 +++++++++----- mitmproxy/contrib/kaitaistruct/jpeg.py | 16 ++++++++---- mitmproxy/contrib/kaitaistruct/make.sh | 10 ++++++++ mitmproxy/contrib/kaitaistruct/png.py | 26 ++++++++++++-------- setup.py | 2 +- 10 files changed, 74 insertions(+), 31 deletions(-) create mode 100755 mitmproxy/contrib/kaitaistruct/make.sh diff --git a/.gitignore b/.gitignore index c289ed503..915a81b44 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ MANIFEST .cache/ .tox*/ build/ +mitmproxy/contrib/kaitaistruct/*.ksy # UI diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py index 062fb38e3..7c74669af 100644 --- a/mitmproxy/contentviews/image/image_parser.py +++ b/mitmproxy/contentviews/image/image_parser.py @@ -37,7 +37,7 @@ def parse_gif(data: bytes) -> Metadata: descriptor = img.logical_screen_descriptor parts = [ ('Format', 'Compuserve GIF'), - ('Version', "GIF{}".format(img.header.version.decode('ASCII'))), + ('Version', "GIF{}".format(img.hdr.version)), ('Size', "{} x {} px".format(descriptor.screen_width, descriptor.screen_height)), ('background', str(descriptor.bg_color_index)) ] diff --git a/mitmproxy/contrib/kaitaistruct/exif.py b/mitmproxy/contrib/kaitaistruct/exif.py index 6236a7086..2e8f1f9ff 100644 --- a/mitmproxy/contrib/kaitaistruct/exif.py +++ b/mitmproxy/contrib/kaitaistruct/exif.py @@ -1,16 +1,18 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was exif.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/exif.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO -from .exif_le import ExifLe -from .exif_be import ExifBe +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + +from exif_le import ExifLe class Exif(KaitaiStruct): def __init__(self, _io, _parent=None, _root=None): self._io = _io @@ -22,3 +24,5 @@ class Exif(KaitaiStruct): self.body = ExifLe(self._io) elif _on == 19789: self.body = ExifBe(self._io) + + diff --git a/mitmproxy/contrib/kaitaistruct/exif_be.py b/mitmproxy/contrib/kaitaistruct/exif_be.py index 7980a9e8f..8a6e7a2b5 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_be.py +++ b/mitmproxy/contrib/kaitaistruct/exif_be.py @@ -1,14 +1,17 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was exif_be.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/exif_be.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + class ExifBe(KaitaiStruct): def __init__(self, _io, _parent=None, _root=None): self._io = _io @@ -569,3 +572,5 @@ class ExifBe(KaitaiStruct): self._m_ifd0 = self._root.Ifd(self._io, self, self._root) self._io.seek(_pos) return self._m_ifd0 if hasattr(self, '_m_ifd0') else None + + diff --git a/mitmproxy/contrib/kaitaistruct/exif_le.py b/mitmproxy/contrib/kaitaistruct/exif_le.py index 207b3bebe..84e53a38d 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_le.py +++ b/mitmproxy/contrib/kaitaistruct/exif_le.py @@ -1,14 +1,17 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was exif_le.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/exif_le.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + class ExifLe(KaitaiStruct): def __init__(self, _io, _parent=None, _root=None): self._io = _io @@ -569,3 +572,5 @@ class ExifLe(KaitaiStruct): self._m_ifd0 = self._root.Ifd(self._io, self, self._root) self._io.seek(_pos) return self._m_ifd0 if hasattr(self, '_m_ifd0') else None + + diff --git a/mitmproxy/contrib/kaitaistruct/gif.py b/mitmproxy/contrib/kaitaistruct/gif.py index 61499cc73..820df568c 100644 --- a/mitmproxy/contrib/kaitaistruct/gif.py +++ b/mitmproxy/contrib/kaitaistruct/gif.py @@ -1,14 +1,17 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was png.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/562154250bea0081fed4e232751b934bc270a0c7/image/gif.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + class Gif(KaitaiStruct): class BlockType(Enum): @@ -24,8 +27,8 @@ class Gif(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.header = self._root.Header(self._io, self, self._root) - self.logical_screen_descriptor = self._root.LogicalScreenDescriptor(self._io, self, self._root) + self.hdr = self._root.Header(self._io, self, self._root) + self.logical_screen_descriptor = self._root.LogicalScreenDescriptorStruct(self._io, self, self._root) if self.logical_screen_descriptor.has_color_table: self._raw_global_color_table = self._io.read_bytes((self.logical_screen_descriptor.color_table_size * 3)) io = KaitaiStream(BytesIO(self._raw_global_color_table)) @@ -55,7 +58,7 @@ class Gif(KaitaiStruct): self.blue = self._io.read_u1() - class LogicalScreenDescriptor(KaitaiStruct): + class LogicalScreenDescriptorStruct(KaitaiStruct): def __init__(self, _io, _parent=None, _root=None): self._io = _io self._parent = _parent @@ -163,7 +166,7 @@ class Gif(KaitaiStruct): self._parent = _parent self._root = _root if _root else self self.magic = self._io.ensure_fixed_contents(struct.pack('3b', 71, 73, 70)) - self.version = self._io.read_bytes(3) + self.version = (self._io.read_bytes(3)).decode(u"ASCII") class ExtGraphicControl(KaitaiStruct): @@ -245,3 +248,6 @@ class Gif(KaitaiStruct): self.body = self._root.ExtGraphicControl(self._io, self, self._root) else: self.body = self._root.Subblocks(self._io, self, self._root) + + + diff --git a/mitmproxy/contrib/kaitaistruct/jpeg.py b/mitmproxy/contrib/kaitaistruct/jpeg.py index 08e382a97..a49c83b93 100644 --- a/mitmproxy/contrib/kaitaistruct/jpeg.py +++ b/mitmproxy/contrib/kaitaistruct/jpeg.py @@ -1,15 +1,18 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was jpeg.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/jpeg.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO -from .exif import Exif +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + +from exif import Exif class Jpeg(KaitaiStruct): class ComponentId(Enum): @@ -127,7 +130,7 @@ class Jpeg(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.magic = self._io.read_strz("ASCII", 0, False, True, True) + self.magic = (self._io.read_bytes_term(0, False, True, True)).decode(u"ASCII") _on = self.magic if _on == u"Exif": self.body = self._root.ExifInJpeg(self._io, self, self._root) @@ -195,7 +198,7 @@ class Jpeg(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.magic = self._io.read_str_byte_limit(5, "ASCII") + self.magic = (self._io.read_bytes(5)).decode(u"ASCII") self.version_major = self._io.read_u1() self.version_minor = self._io.read_u1() self.density_units = self._root.SegmentApp0.DensityUnit(self._io.read_u1()) @@ -204,3 +207,6 @@ class Jpeg(KaitaiStruct): self.thumbnail_x = self._io.read_u1() self.thumbnail_y = self._io.read_u1() self.thumbnail = self._io.read_bytes(((self.thumbnail_x * self.thumbnail_y) * 3)) + + + diff --git a/mitmproxy/contrib/kaitaistruct/make.sh b/mitmproxy/contrib/kaitaistruct/make.sh new file mode 100755 index 000000000..218d51989 --- /dev/null +++ b/mitmproxy/contrib/kaitaistruct/make.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/exif_be.ksy +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/exif_le.ksy +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/exif.ksy +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/gif.ksy +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/jpeg.ksy +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/png.ksy + +kaitai-struct-compiler --target python --opaque-types=true *.ksy diff --git a/mitmproxy/contrib/kaitaistruct/png.py b/mitmproxy/contrib/kaitaistruct/png.py index 2f3c1a5c3..98a706937 100644 --- a/mitmproxy/contrib/kaitaistruct/png.py +++ b/mitmproxy/contrib/kaitaistruct/png.py @@ -1,14 +1,17 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was png.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/9370c720b7d2ad329102d89bdc880ba6a706ef26/image/png.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + class Png(KaitaiStruct): class ColorType(Enum): @@ -51,7 +54,7 @@ class Png(KaitaiStruct): self._parent = _parent self._root = _root if _root else self self.len = self._io.read_u4be() - self.type = self._io.read_str_byte_limit(4, "UTF-8") + self.type = (self._io.read_bytes(4)).decode(u"UTF-8") _on = self.type if _on == u"iTXt": self._raw_body = self._io.read_bytes(self.len) @@ -194,7 +197,7 @@ class Png(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.keyword = self._io.read_strz("UTF-8", 0, False, True, True) + self.keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"UTF-8") self.compression_method = self._io.read_u1() self._raw_text_datastream = self._io.read_bytes_full() self.text_datastream = zlib.decompress(self._raw_text_datastream) @@ -259,12 +262,12 @@ class Png(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.keyword = self._io.read_strz("UTF-8", 0, False, True, True) + self.keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"UTF-8") self.compression_flag = self._io.read_u1() self.compression_method = self._io.read_u1() - self.language_tag = self._io.read_strz("ASCII", 0, False, True, True) - self.translated_keyword = self._io.read_strz("UTF-8", 0, False, True, True) - self.text = self._io.read_str_eos("UTF-8") + self.language_tag = (self._io.read_bytes_term(0, False, True, True)).decode(u"ASCII") + self.translated_keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"UTF-8") + self.text = (self._io.read_bytes_full()).decode(u"UTF-8") class TextChunk(KaitaiStruct): @@ -272,8 +275,8 @@ class Png(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.keyword = self._io.read_strz("iso8859-1", 0, False, True, True) - self.text = self._io.read_str_eos("iso8859-1") + self.keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"iso8859-1") + self.text = (self._io.read_bytes_full()).decode(u"iso8859-1") class TimeChunk(KaitaiStruct): @@ -287,3 +290,6 @@ class Png(KaitaiStruct): self.hour = self._io.read_u1() self.minute = self._io.read_u1() self.second = self._io.read_u1() + + + diff --git a/setup.py b/setup.py index a758a31bf..cb25454e2 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ setup( "html2text>=2016.1.8, <=2016.9.19", "hyperframe>=5.0, <6", "jsbeautifier>=1.6.3, <1.7", - "kaitaistruct>=0.6, <0.7", + "kaitaistruct>=0.7, <0.8", "passlib>=1.6.5, <1.8", "pyasn1>=0.1.9, <0.3", "pyOpenSSL>=16.0, <17.0", From 83a428e0b712a5283525db911acb68d393c64fcb Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 16 Apr 2017 15:17:29 +0200 Subject: [PATCH 2/2] manually fix imports --- mitmproxy/contrib/kaitaistruct/exif.py | 6 +++--- mitmproxy/contrib/kaitaistruct/jpeg.py | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/mitmproxy/contrib/kaitaistruct/exif.py b/mitmproxy/contrib/kaitaistruct/exif.py index 2e8f1f9ff..d99cceef1 100644 --- a/mitmproxy/contrib/kaitaistruct/exif.py +++ b/mitmproxy/contrib/kaitaistruct/exif.py @@ -12,7 +12,9 @@ from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, if parse_version(ks_version) < parse_version('0.7'): raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) -from exif_le import ExifLe +from .exif_le import ExifLe +from .exif_be import ExifBe + class Exif(KaitaiStruct): def __init__(self, _io, _parent=None, _root=None): self._io = _io @@ -24,5 +26,3 @@ class Exif(KaitaiStruct): self.body = ExifLe(self._io) elif _on == 19789: self.body = ExifBe(self._io) - - diff --git a/mitmproxy/contrib/kaitaistruct/jpeg.py b/mitmproxy/contrib/kaitaistruct/jpeg.py index a49c83b93..33fc012fa 100644 --- a/mitmproxy/contrib/kaitaistruct/jpeg.py +++ b/mitmproxy/contrib/kaitaistruct/jpeg.py @@ -12,7 +12,8 @@ from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, if parse_version(ks_version) < parse_version('0.7'): raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) -from exif import Exif +from .exif import Exif + class Jpeg(KaitaiStruct): class ComponentId(Enum): @@ -207,6 +208,3 @@ class Jpeg(KaitaiStruct): self.thumbnail_x = self._io.read_u1() self.thumbnail_y = self._io.read_u1() self.thumbnail = self._io.read_bytes(((self.thumbnail_x * self.thumbnail_y) * 3)) - - -