From ba144b15bc1a33f71200e54eb4a25894a11d6a53 Mon Sep 17 00:00:00 2001 From: madt1m Date: Fri, 22 Jun 2018 22:43:05 +0200 Subject: [PATCH 01/18] Added initial protobuf definitions --- mitmproxy/io/db.py | 0 mitmproxy/io/proto/connection.proto | 38 +++++++++++++++++++++ mitmproxy/io/proto/http.proto | 51 +++++++++++++++++++++++++++++ mitmproxy/io/protobuf.py | 11 +++++++ 4 files changed, 100 insertions(+) create mode 100644 mitmproxy/io/db.py create mode 100644 mitmproxy/io/proto/connection.proto create mode 100644 mitmproxy/io/proto/http.proto create mode 100644 mitmproxy/io/protobuf.py diff --git a/mitmproxy/io/db.py b/mitmproxy/io/db.py new file mode 100644 index 000000000..e69de29bb diff --git a/mitmproxy/io/proto/connection.proto b/mitmproxy/io/proto/connection.proto new file mode 100644 index 000000000..fdcb0d137 --- /dev/null +++ b/mitmproxy/io/proto/connection.proto @@ -0,0 +1,38 @@ +syntax='proto2'; + +message Address { + optional string host = 1; + optional int32 port = 2; +} + +message ClientConnection { + optional string id = 1; + optional Address address = 2; + optional bool tls_established = 3; + optional string clientcert = 4; + optional string mitmcert = 5; + optional float timestamp_start = 6; + optional float timestamp_tls_setup = 7; + optional float timestamp_end = 8; + optional string sni = 9; + optional string cipher_name = 10; + optional bytes alpn_proto_negotiated = 11; + optional string tls_version = 12; +} + +message ServerConnection { + optional string id = 1; + optional Address address = 2; + optional Address ip_address = 3; + optional Address source_address = 4; + optional bool tls_established = 5; + optional string cert = 6; + optional string sni = 7; + optional bytes alpn_proto_negotiated = 8; + optional string tls_version = 9; + optional float timestamp_start = 10; + optional float timestamp_tcp_setup = 11; + optional float timestamp_tls_setup = 12; + optional float timestamp_end = 13; +} + diff --git a/mitmproxy/io/proto/http.proto b/mitmproxy/io/proto/http.proto new file mode 100644 index 000000000..f1b90f78b --- /dev/null +++ b/mitmproxy/io/proto/http.proto @@ -0,0 +1,51 @@ +syntax='proto2'; +import "connection.proto"; + +message HTTPFlow { + optional HTTPRequest request = 1; + optional HTTPResponse response = 2; + optional HTTPError error = 3; + optional ClientConnection client_conn = 4; + optional ServerConnection server_conn = 5; + optional bool intercepted = 6; + optional bool marked = 7; + optional string mode = 8; + optional string id = 9; + optional int32 version = 10; +} + +message HTTPRequest { + optional string first_line_format = 1; + optional bytes method = 2; + optional bytes scheme = 3; + optional bytes host = 4; + optional int32 port = 5; + optional bytes path = 6; + optional bytes http_version = 7; + repeated HTTPHeader headers = 8; + optional bytes content = 9; + optional int64 timestamp_start = 10; + optional int64 timestamp_end = 11; + optional bool is_replay = 12; +} + +message HTTPResponse { + optional bytes http_version = 1; + optional int32 status_code = 2; + optional bytes reason = 3; + repeated HTTPHeader headers = 4; + optional bytes content = 5; + optional int64 timestamp_start = 6; + optional int64 timestamp_end = 7; + optional bool is_replay = 8; +} + +message HTTPError { + optional string msg = 1; + optional int64 timestamp = 2; +} + +message HTTPHeader { + optional bytes name = 1; + optional bytes value = 2; +} \ No newline at end of file diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py new file mode 100644 index 000000000..2dd0f1e7d --- /dev/null +++ b/mitmproxy/io/protobuf.py @@ -0,0 +1,11 @@ + + +"Just define dumps and loads. Files will be handled later. Or maybe...do it, and DONT use it." + +import typing + +def dumps(value) -> bytes: + pass + +def dump(value, file_handle: typing.BinaryIO) -> None: + pass From e842aa3798ee71812cc68ce0f3b7360edbc023c3 Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 23 Jun 2018 03:02:39 +0200 Subject: [PATCH 02/18] Added protobuf methods; implemented initial DB logic --- examples/addons/protodumper.py | 5 + mitmproxy/io/db.py | 32 ++ mitmproxy/io/proto/connection.proto | 38 -- mitmproxy/io/proto/http.proto | 68 ++- mitmproxy/io/proto/http_pb2.py | 724 ++++++++++++++++++++++++++++ mitmproxy/io/protobuf.py | 60 ++- setup.py | 1 + 7 files changed, 868 insertions(+), 60 deletions(-) create mode 100644 examples/addons/protodumper.py delete mode 100644 mitmproxy/io/proto/connection.proto create mode 100644 mitmproxy/io/proto/http_pb2.py diff --git a/examples/addons/protodumper.py b/examples/addons/protodumper.py new file mode 100644 index 000000000..6a1d676c4 --- /dev/null +++ b/examples/addons/protodumper.py @@ -0,0 +1,5 @@ +from mitmproxy.io import protobuf + + +def response(f): + protobuf.loads(protobuf.dumps(f)) diff --git a/mitmproxy/io/db.py b/mitmproxy/io/db.py index e69de29bb..f7e13e915 100644 --- a/mitmproxy/io/db.py +++ b/mitmproxy/io/db.py @@ -0,0 +1,32 @@ +import sqlite3 +from mitmproxy.io import protobuf + + +class DbHandler: + + """ + This class is wrapping up connection to SQLITE DB. + """ + + def __init__(self, db_path="tmp.sqlite"): + self.db_path = db_path + self._con = sqlite3.connect(self.db_path) + self._c = self._con.cursor() + self._create_db() + + def _create_db(self): + with self._con: + self._con.execute('CREATE TABLE IF NOT EXISTS FLOWS(' + 'id INTEGER PRIMARY KEY AUTOINCREMENT,' + 'pbuf_blob BLOB)') + + def store(self, flows): + blobs = [] + for flow in flows: + blobs.append(protobuf.dumps(flow)) + with self._con: + self._con.executemany('INSERT INTO FLOWS values (?)', blobs) + + def load(self): + self._c.execute('SELECT * FROM FLOWS') + return self._c.fetchall() diff --git a/mitmproxy/io/proto/connection.proto b/mitmproxy/io/proto/connection.proto deleted file mode 100644 index fdcb0d137..000000000 --- a/mitmproxy/io/proto/connection.proto +++ /dev/null @@ -1,38 +0,0 @@ -syntax='proto2'; - -message Address { - optional string host = 1; - optional int32 port = 2; -} - -message ClientConnection { - optional string id = 1; - optional Address address = 2; - optional bool tls_established = 3; - optional string clientcert = 4; - optional string mitmcert = 5; - optional float timestamp_start = 6; - optional float timestamp_tls_setup = 7; - optional float timestamp_end = 8; - optional string sni = 9; - optional string cipher_name = 10; - optional bytes alpn_proto_negotiated = 11; - optional string tls_version = 12; -} - -message ServerConnection { - optional string id = 1; - optional Address address = 2; - optional Address ip_address = 3; - optional Address source_address = 4; - optional bool tls_established = 5; - optional string cert = 6; - optional string sni = 7; - optional bytes alpn_proto_negotiated = 8; - optional string tls_version = 9; - optional float timestamp_start = 10; - optional float timestamp_tcp_setup = 11; - optional float timestamp_tls_setup = 12; - optional float timestamp_end = 13; -} - diff --git a/mitmproxy/io/proto/http.proto b/mitmproxy/io/proto/http.proto index f1b90f78b..3db5333cb 100644 --- a/mitmproxy/io/proto/http.proto +++ b/mitmproxy/io/proto/http.proto @@ -1,5 +1,4 @@ syntax='proto2'; -import "connection.proto"; message HTTPFlow { optional HTTPRequest request = 1; @@ -16,36 +15,73 @@ message HTTPFlow { message HTTPRequest { optional string first_line_format = 1; - optional bytes method = 2; - optional bytes scheme = 3; - optional bytes host = 4; + optional string method = 2; + optional string scheme = 3; + optional string host = 4; optional int32 port = 5; - optional bytes path = 6; - optional bytes http_version = 7; + optional string path = 6; + optional string http_version = 7; repeated HTTPHeader headers = 8; optional bytes content = 9; - optional int64 timestamp_start = 10; - optional int64 timestamp_end = 11; + optional float timestamp_start = 10; + optional float timestamp_end = 11; optional bool is_replay = 12; } message HTTPResponse { - optional bytes http_version = 1; + optional string http_version = 1; optional int32 status_code = 2; - optional bytes reason = 3; + optional string reason = 3; repeated HTTPHeader headers = 4; optional bytes content = 5; - optional int64 timestamp_start = 6; - optional int64 timestamp_end = 7; + optional float timestamp_start = 6; + optional float timestamp_end = 7; optional bool is_replay = 8; } message HTTPError { optional string msg = 1; - optional int64 timestamp = 2; + optional float timestamp = 2; } message HTTPHeader { - optional bytes name = 1; - optional bytes value = 2; -} \ No newline at end of file + optional string name = 1; + optional string value = 2; +} + + +message Address { + optional string host = 1; + optional int32 port = 2; +} + +message ClientConnection { + optional string id = 1; + optional Address address = 2; + optional bool tls_established = 3; + optional string clientcert = 4; + optional string mitmcert = 5; + optional float timestamp_start = 6; + optional float timestamp_tls_setup = 7; + optional float timestamp_end = 8; + optional string sni = 9; + optional string cipher_name = 10; + optional bytes alpn_proto_negotiated = 11; + optional string tls_version = 12; +} + +message ServerConnection { + optional string id = 1; + optional Address address = 2; + optional Address ip_address = 3; + optional Address source_address = 4; + optional bool tls_established = 5; + optional string cert = 6; + optional string sni = 7; + optional bytes alpn_proto_negotiated = 8; + optional string tls_version = 9; + optional float timestamp_start = 10; + optional float timestamp_tcp_setup = 11; + optional float timestamp_tls_setup = 12; + optional float timestamp_end = 13; +} diff --git a/mitmproxy/io/proto/http_pb2.py b/mitmproxy/io/proto/http_pb2.py new file mode 100644 index 000000000..b1bcfbb6a --- /dev/null +++ b/mitmproxy/io/proto/http_pb2.py @@ -0,0 +1,724 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: http.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='http.proto', + package='', + syntax='proto2', + serialized_pb=_b('\n\nhttp.proto\"\x85\x02\n\x08HTTPFlow\x12\x1d\n\x07request\x18\x01 \x01(\x0b\x32\x0c.HTTPRequest\x12\x1f\n\x08response\x18\x02 \x01(\x0b\x32\r.HTTPResponse\x12\x19\n\x05\x65rror\x18\x03 \x01(\x0b\x32\n.HTTPError\x12&\n\x0b\x63lient_conn\x18\x04 \x01(\x0b\x32\x11.ClientConnection\x12&\n\x0bserver_conn\x18\x05 \x01(\x0b\x32\x11.ServerConnection\x12\x13\n\x0bintercepted\x18\x06 \x01(\x08\x12\x0e\n\x06marked\x18\x07 \x01(\x08\x12\x0c\n\x04mode\x18\x08 \x01(\t\x12\n\n\x02id\x18\t \x01(\t\x12\x0f\n\x07version\x18\n \x01(\x05\"\xfa\x01\n\x0bHTTPRequest\x12\x19\n\x11\x66irst_line_format\x18\x01 \x01(\t\x12\x0e\n\x06method\x18\x02 \x01(\t\x12\x0e\n\x06scheme\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\x05\x12\x0c\n\x04path\x18\x06 \x01(\t\x12\x14\n\x0chttp_version\x18\x07 \x01(\t\x12\x1c\n\x07headers\x18\x08 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\t \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x15\n\rtimestamp_end\x18\x0b \x01(\x02\x12\x11\n\tis_replay\x18\x0c \x01(\x08\"\xbb\x01\n\x0cHTTPResponse\x12\x14\n\x0chttp_version\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x1c\n\x07headers\x18\x04 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x07 \x01(\x02\x12\x11\n\tis_replay\x18\x08 \x01(\x08\"+\n\tHTTPError\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x02\")\n\nHTTPHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"%\n\x07\x41\x64\x64ress\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\x9b\x02\n\x10\x43lientConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x03 \x01(\x08\x12\x12\n\nclientcert\x18\x04 \x01(\t\x12\x10\n\x08mitmcert\x18\x05 \x01(\t\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x07 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x08 \x01(\x02\x12\x0b\n\x03sni\x18\t \x01(\t\x12\x13\n\x0b\x63ipher_name\x18\n \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x0b \x01(\x0c\x12\x13\n\x0btls_version\x18\x0c \x01(\t\"\xcb\x02\n\x10ServerConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x1c\n\nip_address\x18\x03 \x01(\x0b\x32\x08.Address\x12 \n\x0esource_address\x18\x04 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x05 \x01(\x08\x12\x0c\n\x04\x63\x65rt\x18\x06 \x01(\t\x12\x0b\n\x03sni\x18\x07 \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x08 \x01(\x0c\x12\x13\n\x0btls_version\x18\t \x01(\t\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x1b\n\x13timestamp_tcp_setup\x18\x0b \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x0c \x01(\x02\x12\x15\n\rtimestamp_end\x18\r \x01(\x02') +) + + + + +_HTTPFLOW = _descriptor.Descriptor( + name='HTTPFlow', + full_name='HTTPFlow', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='request', full_name='HTTPFlow.request', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='response', full_name='HTTPFlow.response', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='error', full_name='HTTPFlow.error', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='client_conn', full_name='HTTPFlow.client_conn', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='server_conn', full_name='HTTPFlow.server_conn', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='intercepted', full_name='HTTPFlow.intercepted', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='marked', full_name='HTTPFlow.marked', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mode', full_name='HTTPFlow.mode', index=7, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='id', full_name='HTTPFlow.id', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='version', full_name='HTTPFlow.version', index=9, + number=10, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=15, + serialized_end=276, +) + + +_HTTPREQUEST = _descriptor.Descriptor( + name='HTTPRequest', + full_name='HTTPRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='first_line_format', full_name='HTTPRequest.first_line_format', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='method', full_name='HTTPRequest.method', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='scheme', full_name='HTTPRequest.scheme', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='host', full_name='HTTPRequest.host', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='port', full_name='HTTPRequest.port', index=4, + number=5, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='path', full_name='HTTPRequest.path', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='http_version', full_name='HTTPRequest.http_version', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='headers', full_name='HTTPRequest.headers', index=7, + number=8, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='content', full_name='HTTPRequest.content', index=8, + number=9, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='HTTPRequest.timestamp_start', index=9, + number=10, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='HTTPRequest.timestamp_end', index=10, + number=11, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='is_replay', full_name='HTTPRequest.is_replay', index=11, + number=12, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=279, + serialized_end=529, +) + + +_HTTPRESPONSE = _descriptor.Descriptor( + name='HTTPResponse', + full_name='HTTPResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='http_version', full_name='HTTPResponse.http_version', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='status_code', full_name='HTTPResponse.status_code', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='reason', full_name='HTTPResponse.reason', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='headers', full_name='HTTPResponse.headers', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='content', full_name='HTTPResponse.content', index=4, + number=5, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='HTTPResponse.timestamp_start', index=5, + number=6, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='HTTPResponse.timestamp_end', index=6, + number=7, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='is_replay', full_name='HTTPResponse.is_replay', index=7, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=532, + serialized_end=719, +) + + +_HTTPERROR = _descriptor.Descriptor( + name='HTTPError', + full_name='HTTPError', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='msg', full_name='HTTPError.msg', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp', full_name='HTTPError.timestamp', index=1, + number=2, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=721, + serialized_end=764, +) + + +_HTTPHEADER = _descriptor.Descriptor( + name='HTTPHeader', + full_name='HTTPHeader', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='HTTPHeader.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='HTTPHeader.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=766, + serialized_end=807, +) + + +_ADDRESS = _descriptor.Descriptor( + name='Address', + full_name='Address', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='host', full_name='Address.host', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='port', full_name='Address.port', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=809, + serialized_end=846, +) + + +_CLIENTCONNECTION = _descriptor.Descriptor( + name='ClientConnection', + full_name='ClientConnection', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='ClientConnection.id', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='address', full_name='ClientConnection.address', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_established', full_name='ClientConnection.tls_established', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='clientcert', full_name='ClientConnection.clientcert', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mitmcert', full_name='ClientConnection.mitmcert', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='ClientConnection.timestamp_start', index=5, + number=6, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_tls_setup', full_name='ClientConnection.timestamp_tls_setup', index=6, + number=7, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='ClientConnection.timestamp_end', index=7, + number=8, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='sni', full_name='ClientConnection.sni', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='cipher_name', full_name='ClientConnection.cipher_name', index=9, + number=10, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='alpn_proto_negotiated', full_name='ClientConnection.alpn_proto_negotiated', index=10, + number=11, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_version', full_name='ClientConnection.tls_version', index=11, + number=12, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=849, + serialized_end=1132, +) + + +_SERVERCONNECTION = _descriptor.Descriptor( + name='ServerConnection', + full_name='ServerConnection', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='ServerConnection.id', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='address', full_name='ServerConnection.address', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='ip_address', full_name='ServerConnection.ip_address', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='source_address', full_name='ServerConnection.source_address', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_established', full_name='ServerConnection.tls_established', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='cert', full_name='ServerConnection.cert', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='sni', full_name='ServerConnection.sni', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='alpn_proto_negotiated', full_name='ServerConnection.alpn_proto_negotiated', index=7, + number=8, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_version', full_name='ServerConnection.tls_version', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='ServerConnection.timestamp_start', index=9, + number=10, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_tcp_setup', full_name='ServerConnection.timestamp_tcp_setup', index=10, + number=11, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_tls_setup', full_name='ServerConnection.timestamp_tls_setup', index=11, + number=12, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='ServerConnection.timestamp_end', index=12, + number=13, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1135, + serialized_end=1466, +) + +_HTTPFLOW.fields_by_name['request'].message_type = _HTTPREQUEST +_HTTPFLOW.fields_by_name['response'].message_type = _HTTPRESPONSE +_HTTPFLOW.fields_by_name['error'].message_type = _HTTPERROR +_HTTPFLOW.fields_by_name['client_conn'].message_type = _CLIENTCONNECTION +_HTTPFLOW.fields_by_name['server_conn'].message_type = _SERVERCONNECTION +_HTTPREQUEST.fields_by_name['headers'].message_type = _HTTPHEADER +_HTTPRESPONSE.fields_by_name['headers'].message_type = _HTTPHEADER +_CLIENTCONNECTION.fields_by_name['address'].message_type = _ADDRESS +_SERVERCONNECTION.fields_by_name['address'].message_type = _ADDRESS +_SERVERCONNECTION.fields_by_name['ip_address'].message_type = _ADDRESS +_SERVERCONNECTION.fields_by_name['source_address'].message_type = _ADDRESS +DESCRIPTOR.message_types_by_name['HTTPFlow'] = _HTTPFLOW +DESCRIPTOR.message_types_by_name['HTTPRequest'] = _HTTPREQUEST +DESCRIPTOR.message_types_by_name['HTTPResponse'] = _HTTPRESPONSE +DESCRIPTOR.message_types_by_name['HTTPError'] = _HTTPERROR +DESCRIPTOR.message_types_by_name['HTTPHeader'] = _HTTPHEADER +DESCRIPTOR.message_types_by_name['Address'] = _ADDRESS +DESCRIPTOR.message_types_by_name['ClientConnection'] = _CLIENTCONNECTION +DESCRIPTOR.message_types_by_name['ServerConnection'] = _SERVERCONNECTION +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +HTTPFlow = _reflection.GeneratedProtocolMessageType('HTTPFlow', (_message.Message,), dict( + DESCRIPTOR = _HTTPFLOW, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPFlow) + )) +_sym_db.RegisterMessage(HTTPFlow) + +HTTPRequest = _reflection.GeneratedProtocolMessageType('HTTPRequest', (_message.Message,), dict( + DESCRIPTOR = _HTTPREQUEST, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPRequest) + )) +_sym_db.RegisterMessage(HTTPRequest) + +HTTPResponse = _reflection.GeneratedProtocolMessageType('HTTPResponse', (_message.Message,), dict( + DESCRIPTOR = _HTTPRESPONSE, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPResponse) + )) +_sym_db.RegisterMessage(HTTPResponse) + +HTTPError = _reflection.GeneratedProtocolMessageType('HTTPError', (_message.Message,), dict( + DESCRIPTOR = _HTTPERROR, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPError) + )) +_sym_db.RegisterMessage(HTTPError) + +HTTPHeader = _reflection.GeneratedProtocolMessageType('HTTPHeader', (_message.Message,), dict( + DESCRIPTOR = _HTTPHEADER, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPHeader) + )) +_sym_db.RegisterMessage(HTTPHeader) + +Address = _reflection.GeneratedProtocolMessageType('Address', (_message.Message,), dict( + DESCRIPTOR = _ADDRESS, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:Address) + )) +_sym_db.RegisterMessage(Address) + +ClientConnection = _reflection.GeneratedProtocolMessageType('ClientConnection', (_message.Message,), dict( + DESCRIPTOR = _CLIENTCONNECTION, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:ClientConnection) + )) +_sym_db.RegisterMessage(ClientConnection) + +ServerConnection = _reflection.GeneratedProtocolMessageType('ServerConnection', (_message.Message,), dict( + DESCRIPTOR = _SERVERCONNECTION, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:ServerConnection) + )) +_sym_db.RegisterMessage(ServerConnection) + + +# @@protoc_insertion_point(module_scope) diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py index 2dd0f1e7d..dc526d360 100644 --- a/mitmproxy/io/protobuf.py +++ b/mitmproxy/io/protobuf.py @@ -1,11 +1,59 @@ +from mitmproxy import flow +from mitmproxy import exceptions +from mitmproxy.io.proto import http_pb2 -"Just define dumps and loads. Files will be handled later. Or maybe...do it, and DONT use it." +def dumps(f: flow.Flow) -> bytes: + if f.type != "http": + raise exceptions.TypeError("Flow types different than HTTP not supported yet!") + else: + hf = http_pb2.HTTPFlow() + state = f.get_state() + for r in ['request', 'response']: + for field in state[r]: + if hasattr(getattr(hf, r), field) and state[r][field] is not None: + if field == 'headers': + for n, v in state[r][field]: + header = (getattr(hf, r)).headers.add() + header.name = n + header.value = v + else: + setattr(getattr(hf, r), field, state[r][field]) + if state['error']: + hf.error.msg = state['error']['msg'] + hf.error.timestamp = state['error']['timestamp'] + for c in ['client_conn', 'server_conn']: + for field in state[c]: + if hasattr(getattr(hf, c), field) and state[c][field] is not None: + if field in ['address', 'ip_address', 'source_address']: + getattr(getattr(hf, c), field).host = state[c][field][0] + getattr(getattr(hf, c), field).port = state[c][field][1] + else: + setattr(getattr(hf, c), field, state[c][field]) + for val in ['intercepted', 'marked', 'mode', 'id', 'version']: + if state[val] is not None: + setattr(hf, val, state[val]) + return hf.SerializeToString() -import typing -def dumps(value) -> bytes: - pass +def loads(blob: bytes) -> flow.Flow: + r = http_pb2.HTTPFlow() + state = dict() + r.ParseFromString(blob) + _dump_object(r, state) -def dump(value, file_handle: typing.BinaryIO) -> None: - pass + +def _dump_object(obj, d): + for descriptor in obj.DESCRIPTOR.fields: + value = getattr(obj, descriptor.name) + if descriptor.type == descriptor.TYPE_MESSAGE: + d[descriptor.name] = {} + if descriptor.label == descriptor.LABEL_REPEATED: + [_dump_object(v, d[descriptor.name]) for v in value] + else: + _dump_object(value, d[descriptor.name]) + elif descriptor.type == descriptor.TYPE_ENUM: + enum_name = descriptor.enum_type.values[value].name + d[descriptor.name] = enum_name + else: + d[descriptor.name] = value diff --git a/setup.py b/setup.py index 6ba9391e2..acd3f2952 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ setup( "pyOpenSSL>=17.5,<18.1", "pyparsing>=2.1.3, <2.3", "pyperclip>=1.6.0, <1.7", + "protobuf>=3.6.0, <3.7", "ruamel.yaml>=0.13.2, <0.16", "sortedcontainers>=1.5.4,<2.1", "tornado>=4.3,<5.1", From b686073f56667f7caf11c9013d592a8f091ccd9d Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 23 Jun 2018 03:36:58 +0200 Subject: [PATCH 03/18] Hooking the view to load from new DB handler --- examples/addons/protodumper.py | 5 ----- mitmproxy/addons/view.py | 13 +++++-------- mitmproxy/io/__init__.py | 3 ++- mitmproxy/io/db.py | 22 ++++++++++++++++------ 4 files changed, 23 insertions(+), 20 deletions(-) delete mode 100644 examples/addons/protodumper.py diff --git a/examples/addons/protodumper.py b/examples/addons/protodumper.py deleted file mode 100644 index 6a1d676c4..000000000 --- a/examples/addons/protodumper.py +++ /dev/null @@ -1,5 +0,0 @@ -from mitmproxy.io import protobuf - - -def response(f): - protobuf.loads(protobuf.dumps(f)) diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 1c8bd0cee..3730aa8d6 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -447,16 +447,13 @@ class View(collections.Sequence): Load flows into the view, without processing them with addons. """ try: - with open(path, "rb") as f: - for i in io.FlowReader(f).stream(): - # Do this to get a new ID, so we can load the same file N times and - # get new flows each time. It would be more efficient to just have a - # .newid() method or something. - self.add([i.copy()]) + dh = io.DbHandler(path) + for f in dh.load(): + self.add([f.copy()]) + except exceptions.TypeError as e: + ctx.log.error(str(e)) except IOError as e: ctx.log.error(e.strerror) - except exceptions.FlowReadException as e: - ctx.log.error(str(e)) def add(self, flows: typing.Sequence[mitmproxy.flow.Flow]) -> None: """ diff --git a/mitmproxy/io/__init__.py b/mitmproxy/io/__init__.py index 540e68718..854d25058 100644 --- a/mitmproxy/io/__init__.py +++ b/mitmproxy/io/__init__.py @@ -1,7 +1,8 @@ from .io import FlowWriter, FlowReader, FilteredFlowWriter, read_flows_from_paths +from .db import DbHandler __all__ = [ - "FlowWriter", "FlowReader", "FilteredFlowWriter", "read_flows_from_paths" + "FlowWriter", "FlowReader", "FilteredFlowWriter", "read_flows_from_paths", "DbHandler" ] diff --git a/mitmproxy/io/db.py b/mitmproxy/io/db.py index f7e13e915..b59650ab0 100644 --- a/mitmproxy/io/db.py +++ b/mitmproxy/io/db.py @@ -1,5 +1,9 @@ import sqlite3 +import os + from mitmproxy.io import protobuf +from mitmproxy.http import HTTPFlow +from mitmproxy import exceptions class DbHandler: @@ -8,7 +12,11 @@ class DbHandler: This class is wrapping up connection to SQLITE DB. """ - def __init__(self, db_path="tmp.sqlite"): + def __init__(self, db_path="/tmp/tmp.sqlite"): + if os.path.isfile(db_path): + self.db_path = db_path + else: + raise IOError("Invalid path!") self.db_path = db_path self._con = sqlite3.connect(self.db_path) self._c = self._con.cursor() @@ -17,16 +25,18 @@ class DbHandler: def _create_db(self): with self._con: self._con.execute('CREATE TABLE IF NOT EXISTS FLOWS(' - 'id INTEGER PRIMARY KEY AUTOINCREMENT,' + 'id INTEGER PRIMARY KEY,' 'pbuf_blob BLOB)') def store(self, flows): blobs = [] for flow in flows: - blobs.append(protobuf.dumps(flow)) + blobs.append((protobuf.dumps(flow),)) with self._con: - self._con.executemany('INSERT INTO FLOWS values (?)', blobs) + self._con.executemany('INSERT INTO FLOWS (pbuf_blob) values (?)', blobs) def load(self): - self._c.execute('SELECT * FROM FLOWS') - return self._c.fetchall() + flows = [] + self._c.execute('SELECT pbuf_blob FROM FLOWS') + for row in self._c.fetchall(): + flows.append(HTTPFlow.from_state(protobuf.loads(row[0]))) From 265e1f9e1f4931f2c97ba01a5748e2737f25497f Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 23 Jun 2018 03:52:46 +0200 Subject: [PATCH 04/18] Fixed a bug in protobuf loading -> no return --- mitmproxy/io/db.py | 2 +- mitmproxy/io/protobuf.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mitmproxy/io/db.py b/mitmproxy/io/db.py index b59650ab0..63cb0880a 100644 --- a/mitmproxy/io/db.py +++ b/mitmproxy/io/db.py @@ -39,4 +39,4 @@ class DbHandler: flows = [] self._c.execute('SELECT pbuf_blob FROM FLOWS') for row in self._c.fetchall(): - flows.append(HTTPFlow.from_state(protobuf.loads(row[0]))) + flows.append((protobuf.loads(row[0]))) diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py index dc526d360..bb24cddc9 100644 --- a/mitmproxy/io/protobuf.py +++ b/mitmproxy/io/protobuf.py @@ -1,5 +1,6 @@ from mitmproxy import flow from mitmproxy import exceptions +from mitmproxy.http import HTTPFlow from mitmproxy.io.proto import http_pb2 @@ -41,6 +42,7 @@ def loads(blob: bytes) -> flow.Flow: state = dict() r.ParseFromString(blob) _dump_object(r, state) + return HTTPFlow.from_state(state) def _dump_object(obj, d): From c3f4f2782927d9819fbe9afe33ec4b4ed6b4113d Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 23 Jun 2018 13:03:35 +0200 Subject: [PATCH 05/18] New explicit parsing HTTPFlow into proto native object. Viceversa not yet implemented. --- mitmproxy/io/proto/http.proto | 7 ++ mitmproxy/io/proto/http_pb2.py | 70 ++++++++++++++++++-- mitmproxy/io/protobuf.py | 117 +++++++++++++++++++++++++++++++-- 3 files changed, 185 insertions(+), 9 deletions(-) diff --git a/mitmproxy/io/proto/http.proto b/mitmproxy/io/proto/http.proto index 3db5333cb..30c11a0a7 100644 --- a/mitmproxy/io/proto/http.proto +++ b/mitmproxy/io/proto/http.proto @@ -68,6 +68,7 @@ message ClientConnection { optional string cipher_name = 10; optional bytes alpn_proto_negotiated = 11; optional string tls_version = 12; + repeated TLSExtension tls_extensions = 13; } message ServerConnection { @@ -84,4 +85,10 @@ message ServerConnection { optional float timestamp_tcp_setup = 11; optional float timestamp_tls_setup = 12; optional float timestamp_end = 13; + optional ServerConnection via = 14; } + +message TLSExtension { + optional int64 int = 1; + optional bytes bytes = 2; +} \ No newline at end of file diff --git a/mitmproxy/io/proto/http_pb2.py b/mitmproxy/io/proto/http_pb2.py index b1bcfbb6a..d766a4ccb 100644 --- a/mitmproxy/io/proto/http_pb2.py +++ b/mitmproxy/io/proto/http_pb2.py @@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='http.proto', package='', syntax='proto2', - serialized_pb=_b('\n\nhttp.proto\"\x85\x02\n\x08HTTPFlow\x12\x1d\n\x07request\x18\x01 \x01(\x0b\x32\x0c.HTTPRequest\x12\x1f\n\x08response\x18\x02 \x01(\x0b\x32\r.HTTPResponse\x12\x19\n\x05\x65rror\x18\x03 \x01(\x0b\x32\n.HTTPError\x12&\n\x0b\x63lient_conn\x18\x04 \x01(\x0b\x32\x11.ClientConnection\x12&\n\x0bserver_conn\x18\x05 \x01(\x0b\x32\x11.ServerConnection\x12\x13\n\x0bintercepted\x18\x06 \x01(\x08\x12\x0e\n\x06marked\x18\x07 \x01(\x08\x12\x0c\n\x04mode\x18\x08 \x01(\t\x12\n\n\x02id\x18\t \x01(\t\x12\x0f\n\x07version\x18\n \x01(\x05\"\xfa\x01\n\x0bHTTPRequest\x12\x19\n\x11\x66irst_line_format\x18\x01 \x01(\t\x12\x0e\n\x06method\x18\x02 \x01(\t\x12\x0e\n\x06scheme\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\x05\x12\x0c\n\x04path\x18\x06 \x01(\t\x12\x14\n\x0chttp_version\x18\x07 \x01(\t\x12\x1c\n\x07headers\x18\x08 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\t \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x15\n\rtimestamp_end\x18\x0b \x01(\x02\x12\x11\n\tis_replay\x18\x0c \x01(\x08\"\xbb\x01\n\x0cHTTPResponse\x12\x14\n\x0chttp_version\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x1c\n\x07headers\x18\x04 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x07 \x01(\x02\x12\x11\n\tis_replay\x18\x08 \x01(\x08\"+\n\tHTTPError\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x02\")\n\nHTTPHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"%\n\x07\x41\x64\x64ress\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\x9b\x02\n\x10\x43lientConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x03 \x01(\x08\x12\x12\n\nclientcert\x18\x04 \x01(\t\x12\x10\n\x08mitmcert\x18\x05 \x01(\t\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x07 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x08 \x01(\x02\x12\x0b\n\x03sni\x18\t \x01(\t\x12\x13\n\x0b\x63ipher_name\x18\n \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x0b \x01(\x0c\x12\x13\n\x0btls_version\x18\x0c \x01(\t\"\xcb\x02\n\x10ServerConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x1c\n\nip_address\x18\x03 \x01(\x0b\x32\x08.Address\x12 \n\x0esource_address\x18\x04 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x05 \x01(\x08\x12\x0c\n\x04\x63\x65rt\x18\x06 \x01(\t\x12\x0b\n\x03sni\x18\x07 \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x08 \x01(\x0c\x12\x13\n\x0btls_version\x18\t \x01(\t\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x1b\n\x13timestamp_tcp_setup\x18\x0b \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x0c \x01(\x02\x12\x15\n\rtimestamp_end\x18\r \x01(\x02') + serialized_pb=_b('\n\nhttp.proto\"\x85\x02\n\x08HTTPFlow\x12\x1d\n\x07request\x18\x01 \x01(\x0b\x32\x0c.HTTPRequest\x12\x1f\n\x08response\x18\x02 \x01(\x0b\x32\r.HTTPResponse\x12\x19\n\x05\x65rror\x18\x03 \x01(\x0b\x32\n.HTTPError\x12&\n\x0b\x63lient_conn\x18\x04 \x01(\x0b\x32\x11.ClientConnection\x12&\n\x0bserver_conn\x18\x05 \x01(\x0b\x32\x11.ServerConnection\x12\x13\n\x0bintercepted\x18\x06 \x01(\x08\x12\x0e\n\x06marked\x18\x07 \x01(\x08\x12\x0c\n\x04mode\x18\x08 \x01(\t\x12\n\n\x02id\x18\t \x01(\t\x12\x0f\n\x07version\x18\n \x01(\x05\"\xfa\x01\n\x0bHTTPRequest\x12\x19\n\x11\x66irst_line_format\x18\x01 \x01(\t\x12\x0e\n\x06method\x18\x02 \x01(\t\x12\x0e\n\x06scheme\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\x05\x12\x0c\n\x04path\x18\x06 \x01(\t\x12\x14\n\x0chttp_version\x18\x07 \x01(\t\x12\x1c\n\x07headers\x18\x08 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\t \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x15\n\rtimestamp_end\x18\x0b \x01(\x02\x12\x11\n\tis_replay\x18\x0c \x01(\x08\"\xbb\x01\n\x0cHTTPResponse\x12\x14\n\x0chttp_version\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x1c\n\x07headers\x18\x04 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x07 \x01(\x02\x12\x11\n\tis_replay\x18\x08 \x01(\x08\"+\n\tHTTPError\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x02\")\n\nHTTPHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"%\n\x07\x41\x64\x64ress\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xc2\x02\n\x10\x43lientConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x03 \x01(\x08\x12\x12\n\nclientcert\x18\x04 \x01(\t\x12\x10\n\x08mitmcert\x18\x05 \x01(\t\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x07 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x08 \x01(\x02\x12\x0b\n\x03sni\x18\t \x01(\t\x12\x13\n\x0b\x63ipher_name\x18\n \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x0b \x01(\x0c\x12\x13\n\x0btls_version\x18\x0c \x01(\t\x12%\n\x0etls_extensions\x18\r \x03(\x0b\x32\r.TLSExtension\"\xeb\x02\n\x10ServerConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x1c\n\nip_address\x18\x03 \x01(\x0b\x32\x08.Address\x12 \n\x0esource_address\x18\x04 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x05 \x01(\x08\x12\x0c\n\x04\x63\x65rt\x18\x06 \x01(\t\x12\x0b\n\x03sni\x18\x07 \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x08 \x01(\x0c\x12\x13\n\x0btls_version\x18\t \x01(\t\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x1b\n\x13timestamp_tcp_setup\x18\x0b \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x0c \x01(\x02\x12\x15\n\rtimestamp_end\x18\r \x01(\x02\x12\x1e\n\x03via\x18\x0e \x01(\x0b\x32\x11.ServerConnection\"*\n\x0cTLSExtension\x12\x0b\n\x03int\x18\x01 \x01(\x03\x12\r\n\x05\x62ytes\x18\x02 \x01(\x0c') ) @@ -512,6 +512,13 @@ _CLIENTCONNECTION = _descriptor.Descriptor( message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_extensions', full_name='ClientConnection.tls_extensions', index=12, + number=13, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -525,7 +532,7 @@ _CLIENTCONNECTION = _descriptor.Descriptor( oneofs=[ ], serialized_start=849, - serialized_end=1132, + serialized_end=1171, ) @@ -627,6 +634,13 @@ _SERVERCONNECTION = _descriptor.Descriptor( message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='via', full_name='ServerConnection.via', index=13, + number=14, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -639,8 +653,46 @@ _SERVERCONNECTION = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1135, - serialized_end=1466, + serialized_start=1174, + serialized_end=1537, +) + + +_TLSEXTENSION = _descriptor.Descriptor( + name='TLSExtension', + full_name='TLSExtension', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='int', full_name='TLSExtension.int', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='bytes', full_name='TLSExtension.bytes', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1539, + serialized_end=1581, ) _HTTPFLOW.fields_by_name['request'].message_type = _HTTPREQUEST @@ -651,9 +703,11 @@ _HTTPFLOW.fields_by_name['server_conn'].message_type = _SERVERCONNECTION _HTTPREQUEST.fields_by_name['headers'].message_type = _HTTPHEADER _HTTPRESPONSE.fields_by_name['headers'].message_type = _HTTPHEADER _CLIENTCONNECTION.fields_by_name['address'].message_type = _ADDRESS +_CLIENTCONNECTION.fields_by_name['tls_extensions'].message_type = _TLSEXTENSION _SERVERCONNECTION.fields_by_name['address'].message_type = _ADDRESS _SERVERCONNECTION.fields_by_name['ip_address'].message_type = _ADDRESS _SERVERCONNECTION.fields_by_name['source_address'].message_type = _ADDRESS +_SERVERCONNECTION.fields_by_name['via'].message_type = _SERVERCONNECTION DESCRIPTOR.message_types_by_name['HTTPFlow'] = _HTTPFLOW DESCRIPTOR.message_types_by_name['HTTPRequest'] = _HTTPREQUEST DESCRIPTOR.message_types_by_name['HTTPResponse'] = _HTTPRESPONSE @@ -662,6 +716,7 @@ DESCRIPTOR.message_types_by_name['HTTPHeader'] = _HTTPHEADER DESCRIPTOR.message_types_by_name['Address'] = _ADDRESS DESCRIPTOR.message_types_by_name['ClientConnection'] = _CLIENTCONNECTION DESCRIPTOR.message_types_by_name['ServerConnection'] = _SERVERCONNECTION +DESCRIPTOR.message_types_by_name['TLSExtension'] = _TLSEXTENSION _sym_db.RegisterFileDescriptor(DESCRIPTOR) HTTPFlow = _reflection.GeneratedProtocolMessageType('HTTPFlow', (_message.Message,), dict( @@ -720,5 +775,12 @@ ServerConnection = _reflection.GeneratedProtocolMessageType('ServerConnection', )) _sym_db.RegisterMessage(ServerConnection) +TLSExtension = _reflection.GeneratedProtocolMessageType('TLSExtension', (_message.Message,), dict( + DESCRIPTOR = _TLSEXTENSION, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:TLSExtension) + )) +_sym_db.RegisterMessage(TLSExtension) + # @@protoc_insertion_point(module_scope) diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py index bb24cddc9..491129a89 100644 --- a/mitmproxy/io/protobuf.py +++ b/mitmproxy/io/protobuf.py @@ -1,10 +1,97 @@ from mitmproxy import flow from mitmproxy import exceptions -from mitmproxy.http import HTTPFlow +from mitmproxy.http import HTTPFlow, HTTPResponse, HTTPRequest +from mitmproxy.connections import ClientConnection, ServerConnection from mitmproxy.io.proto import http_pb2 +def _parse_attr(s_obj, d_obj, attrs): + for attr in attrs: + if hasattr(s_obj, attr) and getattr(s_obj, attr) is not None: + setattr(d_obj, attr, getattr(s_obj, attr)) + + +def _parse_http_response(res: HTTPResponse) -> http_pb2.HTTPResponse: + pres = http_pb2.HTTPResponse + _parse_attr(res, pres, ['http_version', 'status_code', 'reason', + 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) + for n, v in res.headers: + header = pres.headers.add() + header.name = n + header.value = v + return pres + + +def _parse_http_request(req: HTTPRequest) -> http_pb2.HTTPRequest: + preq = http_pb2.HTTPRequest() + _parse_attr(req, preq, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content', + 'timestamp_start', 'timestamp_end', 'is_replay']) + for n, v in req.headers: + header = preq.headers.add() + header.name = n + header.value = v + return preq + + +def _parse_http_client(cc: ClientConnection) -> http_pb2.ClientConnection: + pcc = http_pb2.ClientConnection() + _parse_attr(cc, pcc, ['id', 'tls_established', 'timestamp_start', 'timestamp_tls_setup', 'timestamp_end', 'sni', + 'cipher_name', 'alpn_proto_negotiated', 'tls_version']) + for cert in ['clientcert', 'mitmcert']: + if hasattr(cc, cert) and getattr(cc, cert) is not None: + setattr(pcc, cert, getattr(cc, cert).to_pem()) + for extension in cc.tls_extensions: + ext = pcc.tls_extensions.add() + ext.int = extension[0] + ext.bytes = extension[1] + if cc.address: + pcc.address.host = cc.address[0] + pcc.address.port = cc.address[1] + return pcc + + +def _parse_http_server(sc: ServerConnection) -> http_pb2.ServerConnection: + psc = http_pb2.ServerConnection() + _parse_attr(sc, psc, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', + 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) + for addr in ['address', 'ip_address', 'source_address']: + if hasattr(sc, addr) and getattr(sc, addr) is not None: + getattr(psc, addr).host = getattr(sc, addr)[0] + getattr(psc, addr).port = getattr(sc, addr)[1] + psc.cert = sc.cert.to_pem() + if sc.via: + psc.via.MergeFrom(_parse_http_server(sc.via)) + return psc + + +def _parse_http_error(e: flow.Error) -> http_pb2.HTTPError: + pe = http_pb2.HTTPError() + for attr in ['msg', 'timestamp']: + if hasattr(e, attr) and getattr(e, attr) is not None: + setattr(pe, attr, getattr(e, attr)) + return pe + + +def _parse_http(f: HTTPFlow) -> http_pb2.HTTPFlow(): + pf = http_pb2.HTTPFlow() + pf.request.MergeFrom(_parse_http_request(f.request)) + pf.response.MergeFrom(_parse_http_response(f.response)) + pf.client_conn.MergeFrom(_parse_http_client(f.client_conn)) + pf.server_conn.MergeFrom(_parse_http_server(f.server_conn)) + pf.error.MergeFrom(_parse_http_error(f.error)) + _parse_attr(f, pf, ['intercepted', 'marked', 'mode', 'id', 'version']) + return pf + + def dumps(f: flow.Flow) -> bytes: + if f.type != "http": + raise exceptions.TypeError("Flow types different than HTTP not supported yet!") + else: + p = _parse_http(f) + return p.SerializeToString() + + +def dumps_state(f: flow.Flow) -> bytes: if f.type != "http": raise exceptions.TypeError("Flow types different than HTTP not supported yet!") else: @@ -29,6 +116,12 @@ def dumps(f: flow.Flow) -> bytes: if field in ['address', 'ip_address', 'source_address']: getattr(getattr(hf, c), field).host = state[c][field][0] getattr(getattr(hf, c), field).port = state[c][field][1] + elif field == "tls_extensions": + for t in state[c][field]: + if t[0] and t[1]: + ext = getattr(hf, c).tls_extensions.add() + ext.int = t[0] + ext.bytes = t[1] else: setattr(getattr(hf, c), field, state[c][field]) for val in ['intercepted', 'marked', 'mode', 'id', 'version']: @@ -37,11 +130,14 @@ def dumps(f: flow.Flow) -> bytes: return hf.SerializeToString() -def loads(blob: bytes) -> flow.Flow: +def loads_state(blob: bytes) -> flow.Flow: r = http_pb2.HTTPFlow() state = dict() r.ParseFromString(blob) _dump_object(r, state) + # ugly repair for tls_extensions typing + for c in ["client_conn", "server_conn"]: + state[c]['tls_extensions'] = [] return HTTPFlow.from_state(state) @@ -49,13 +145,24 @@ def _dump_object(obj, d): for descriptor in obj.DESCRIPTOR.fields: value = getattr(obj, descriptor.name) if descriptor.type == descriptor.TYPE_MESSAGE: - d[descriptor.name] = {} if descriptor.label == descriptor.LABEL_REPEATED: - [_dump_object(v, d[descriptor.name]) for v in value] + d[descriptor] = [] + if value: + for v in value: + _dump_object(v, d[descriptor.name]) + else: + d[descriptor.name] = None else: + d[descriptor] = {} _dump_object(value, d[descriptor.name]) elif descriptor.type == descriptor.TYPE_ENUM: enum_name = descriptor.enum_type.values[value].name d[descriptor.name] = enum_name else: - d[descriptor.name] = value + if value == "" or value == b"": + d[descriptor.name] = None + else: + if type(d) == list: + d.append([descriptor.name, value]) + elif type(d) == dict: + d[descriptor.name] = value From 0b68597a791869e283453ef70422c9de28823908 Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 23 Jun 2018 18:46:06 +0200 Subject: [PATCH 06/18] Fixed bugs in parsing HTTPFlow objects --- mitmproxy/io/protobuf.py | 47 ++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py index 491129a89..f39489de9 100644 --- a/mitmproxy/io/protobuf.py +++ b/mitmproxy/io/protobuf.py @@ -12,13 +12,14 @@ def _parse_attr(s_obj, d_obj, attrs): def _parse_http_response(res: HTTPResponse) -> http_pb2.HTTPResponse: - pres = http_pb2.HTTPResponse + pres = http_pb2.HTTPResponse() _parse_attr(res, pres, ['http_version', 'status_code', 'reason', 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) - for n, v in res.headers: - header = pres.headers.add() - header.name = n - header.value = v + if res.headers: + for h in res.headers.fields: + header = pres.headers.add() + header.name = h[0] + header.value = h[1] return pres @@ -26,10 +27,11 @@ def _parse_http_request(req: HTTPRequest) -> http_pb2.HTTPRequest: preq = http_pb2.HTTPRequest() _parse_attr(req, preq, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) - for n, v in req.headers: - header = preq.headers.add() - header.name = n - header.value = v + if req.headers: + for h in req.headers.fields: + header = preq.headers.add() + header.name = h[0] + header.value = h[1] return preq @@ -40,10 +42,11 @@ def _parse_http_client(cc: ClientConnection) -> http_pb2.ClientConnection: for cert in ['clientcert', 'mitmcert']: if hasattr(cc, cert) and getattr(cc, cert) is not None: setattr(pcc, cert, getattr(cc, cert).to_pem()) - for extension in cc.tls_extensions: - ext = pcc.tls_extensions.add() - ext.int = extension[0] - ext.bytes = extension[1] + if cc.tls_extensions: + for extension in cc.tls_extensions: + ext = pcc.tls_extensions.add() + ext.int = extension[0] + ext.bytes = extension[1] if cc.address: pcc.address.host = cc.address[0] pcc.address.port = cc.address[1] @@ -58,7 +61,8 @@ def _parse_http_server(sc: ServerConnection) -> http_pb2.ServerConnection: if hasattr(sc, addr) and getattr(sc, addr) is not None: getattr(psc, addr).host = getattr(sc, addr)[0] getattr(psc, addr).port = getattr(sc, addr)[1] - psc.cert = sc.cert.to_pem() + if psc.cert: + psc.cert = sc.cert.to_pem() if sc.via: psc.via.MergeFrom(_parse_http_server(sc.via)) return psc @@ -74,11 +78,16 @@ def _parse_http_error(e: flow.Error) -> http_pb2.HTTPError: def _parse_http(f: HTTPFlow) -> http_pb2.HTTPFlow(): pf = http_pb2.HTTPFlow() - pf.request.MergeFrom(_parse_http_request(f.request)) - pf.response.MergeFrom(_parse_http_response(f.response)) - pf.client_conn.MergeFrom(_parse_http_client(f.client_conn)) - pf.server_conn.MergeFrom(_parse_http_server(f.server_conn)) - pf.error.MergeFrom(_parse_http_error(f.error)) + if f.request: + pf.request.MergeFrom(_parse_http_request(f.request)) + if f.response: + pf.response.MergeFrom(_parse_http_response(f.response)) + if f.client_conn: + pf.client_conn.MergeFrom(_parse_http_client(f.client_conn)) + if f.server_conn: + pf.server_conn.MergeFrom(_parse_http_server(f.server_conn)) + if f.error: + pf.error.MergeFrom(_parse_http_error(f.error)) _parse_attr(f, pf, ['intercepted', 'marked', 'mode', 'id', 'version']) return pf From 72958e17de1a7ec62c3de81a8250cdf32d05456c Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 23 Jun 2018 19:02:31 +0200 Subject: [PATCH 07/18] Improved parsing mechanism for better readability --- mitmproxy/io/protobuf.py | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py index f39489de9..38f9046b7 100644 --- a/mitmproxy/io/protobuf.py +++ b/mitmproxy/io/protobuf.py @@ -1,5 +1,6 @@ from mitmproxy import flow from mitmproxy import exceptions +from mitmproxy import ctx from mitmproxy.http import HTTPFlow, HTTPResponse, HTTPRequest from mitmproxy.connections import ClientConnection, ServerConnection from mitmproxy.io.proto import http_pb2 @@ -76,30 +77,43 @@ def _parse_http_error(e: flow.Error) -> http_pb2.HTTPError: return pe -def _parse_http(f: HTTPFlow) -> http_pb2.HTTPFlow(): +def parse_http(f: HTTPFlow) -> http_pb2.HTTPFlow(): pf = http_pb2.HTTPFlow() - if f.request: - pf.request.MergeFrom(_parse_http_request(f.request)) - if f.response: - pf.response.MergeFrom(_parse_http_response(f.response)) - if f.client_conn: - pf.client_conn.MergeFrom(_parse_http_client(f.client_conn)) - if f.server_conn: - pf.server_conn.MergeFrom(_parse_http_server(f.server_conn)) - if f.error: - pf.error.MergeFrom(_parse_http_error(f.error)) + for p in ['request', 'response', 'client_conn', 'server_conn', 'error']: + if hasattr(f, p): + getattr(pf, p).MergeFrom(parsers[p](getattr(f, p))) _parse_attr(f, pf, ['intercepted', 'marked', 'mode', 'id', 'version']) return pf +parsers = { + 'request': _parse_http_request, + 'response': _parse_http_response, + 'error': _parse_http_error, + 'client_conn': _parse_http_client, + 'server_conn': _parse_http_server +} + + def dumps(f: flow.Flow) -> bytes: if f.type != "http": raise exceptions.TypeError("Flow types different than HTTP not supported yet!") else: - p = _parse_http(f) + p = parse_http(f) return p.SerializeToString() +def loads(b: bytes, type='http') -> flow.Flow: + if type != 'http': + raise exceptions.TypeError("Flow types different than HTTP not supported yet!") + else: + p = http_pb2.HTTPFlow() + try: + p.ParseFromString(b) + except Exception as e: + ctx.log(e.strerror) + + def dumps_state(f: flow.Flow) -> bytes: if f.type != "http": raise exceptions.TypeError("Flow types different than HTTP not supported yet!") From a325731854d066824f74c7e05bccb693f2a3c837 Mon Sep 17 00:00:00 2001 From: Pietro Francesco Tirenna Date: Thu, 28 Jun 2018 08:51:05 +0200 Subject: [PATCH 08/18] Refactored naming strategy --- mitmproxy/io/protobuf.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py index 38f9046b7..f480c69e8 100644 --- a/mitmproxy/io/protobuf.py +++ b/mitmproxy/io/protobuf.py @@ -6,15 +6,15 @@ from mitmproxy.connections import ClientConnection, ServerConnection from mitmproxy.io.proto import http_pb2 -def _parse_attr(s_obj, d_obj, attrs): +def _move_attrs(s_obj, d_obj, attrs): for attr in attrs: if hasattr(s_obj, attr) and getattr(s_obj, attr) is not None: setattr(d_obj, attr, getattr(s_obj, attr)) -def _parse_http_response(res: HTTPResponse) -> http_pb2.HTTPResponse: +def _extract_http_response(res: HTTPResponse) -> http_pb2.HTTPResponse: pres = http_pb2.HTTPResponse() - _parse_attr(res, pres, ['http_version', 'status_code', 'reason', + _move_attrs(res, pres, ['http_version', 'status_code', 'reason', 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) if res.headers: for h in res.headers.fields: @@ -24,9 +24,9 @@ def _parse_http_response(res: HTTPResponse) -> http_pb2.HTTPResponse: return pres -def _parse_http_request(req: HTTPRequest) -> http_pb2.HTTPRequest: +def _extract_http_request(req: HTTPRequest) -> http_pb2.HTTPRequest: preq = http_pb2.HTTPRequest() - _parse_attr(req, preq, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content', + _move_attrs(req, preq, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) if req.headers: for h in req.headers.fields: @@ -36,9 +36,9 @@ def _parse_http_request(req: HTTPRequest) -> http_pb2.HTTPRequest: return preq -def _parse_http_client(cc: ClientConnection) -> http_pb2.ClientConnection: +def _extract_http_client(cc: ClientConnection) -> http_pb2.ClientConnection: pcc = http_pb2.ClientConnection() - _parse_attr(cc, pcc, ['id', 'tls_established', 'timestamp_start', 'timestamp_tls_setup', 'timestamp_end', 'sni', + _move_attrs(cc, pcc, ['id', 'tls_established', 'timestamp_start', 'timestamp_tls_setup', 'timestamp_end', 'sni', 'cipher_name', 'alpn_proto_negotiated', 'tls_version']) for cert in ['clientcert', 'mitmcert']: if hasattr(cc, cert) and getattr(cc, cert) is not None: @@ -54,9 +54,9 @@ def _parse_http_client(cc: ClientConnection) -> http_pb2.ClientConnection: return pcc -def _parse_http_server(sc: ServerConnection) -> http_pb2.ServerConnection: +def _extract_http_server(sc: ServerConnection) -> http_pb2.ServerConnection: psc = http_pb2.ServerConnection() - _parse_attr(sc, psc, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', + _move_attrs(sc, psc, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) for addr in ['address', 'ip_address', 'source_address']: if hasattr(sc, addr) and getattr(sc, addr) is not None: @@ -65,11 +65,11 @@ def _parse_http_server(sc: ServerConnection) -> http_pb2.ServerConnection: if psc.cert: psc.cert = sc.cert.to_pem() if sc.via: - psc.via.MergeFrom(_parse_http_server(sc.via)) + psc.via.MergeFrom(_extract_http_server(sc.via)) return psc -def _parse_http_error(e: flow.Error) -> http_pb2.HTTPError: +def _extract_http_error(e: flow.Error) -> http_pb2.HTTPError: pe = http_pb2.HTTPError() for attr in ['msg', 'timestamp']: if hasattr(e, attr) and getattr(e, attr) is not None: @@ -77,21 +77,21 @@ def _parse_http_error(e: flow.Error) -> http_pb2.HTTPError: return pe -def parse_http(f: HTTPFlow) -> http_pb2.HTTPFlow(): +def extract_http(f: HTTPFlow) -> http_pb2.HTTPFlow(): pf = http_pb2.HTTPFlow() for p in ['request', 'response', 'client_conn', 'server_conn', 'error']: if hasattr(f, p): getattr(pf, p).MergeFrom(parsers[p](getattr(f, p))) - _parse_attr(f, pf, ['intercepted', 'marked', 'mode', 'id', 'version']) + _move_attrs(f, pf, ['intercepted', 'marked', 'mode', 'id', 'version']) return pf parsers = { - 'request': _parse_http_request, - 'response': _parse_http_response, - 'error': _parse_http_error, - 'client_conn': _parse_http_client, - 'server_conn': _parse_http_server + 'request': _extract_http_request, + 'response': _extract_http_response, + 'error': _extract_http_error, + 'client_conn': _extract_http_client, + 'server_conn': _extract_http_server } From ea4af9ce17ecf24c3c02c040201ef74a6fc4fb5b Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 30 Jun 2018 09:27:18 +0200 Subject: [PATCH 09/18] Completed untested version of object loading --- mitmproxy/io/protobuf.py | 192 +++++++++++++++++++-------------------- 1 file changed, 93 insertions(+), 99 deletions(-) diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py index f480c69e8..dd2852869 100644 --- a/mitmproxy/io/protobuf.py +++ b/mitmproxy/io/protobuf.py @@ -2,17 +2,22 @@ from mitmproxy import flow from mitmproxy import exceptions from mitmproxy import ctx from mitmproxy.http import HTTPFlow, HTTPResponse, HTTPRequest +from mitmproxy.certs import Cert from mitmproxy.connections import ClientConnection, ServerConnection from mitmproxy.io.proto import http_pb2 def _move_attrs(s_obj, d_obj, attrs): for attr in attrs: - if hasattr(s_obj, attr) and getattr(s_obj, attr) is not None: - setattr(d_obj, attr, getattr(s_obj, attr)) + if not isinstance(d_obj, dict): + if hasattr(s_obj, attr) and getattr(s_obj, attr) is not None: + setattr(d_obj, attr, getattr(s_obj, attr)) + else: + if hasattr(s_obj, attr) and getattr(s_obj, attr) is not None: + d_obj[attr] = getattr(s_obj, attr) -def _extract_http_response(res: HTTPResponse) -> http_pb2.HTTPResponse: +def _dump_http_response(res: HTTPResponse) -> http_pb2.HTTPResponse: pres = http_pb2.HTTPResponse() _move_attrs(res, pres, ['http_version', 'status_code', 'reason', 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) @@ -24,7 +29,7 @@ def _extract_http_response(res: HTTPResponse) -> http_pb2.HTTPResponse: return pres -def _extract_http_request(req: HTTPRequest) -> http_pb2.HTTPRequest: +def _dump_http_request(req: HTTPRequest) -> http_pb2.HTTPRequest: preq = http_pb2.HTTPRequest() _move_attrs(req, preq, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) @@ -36,7 +41,7 @@ def _extract_http_request(req: HTTPRequest) -> http_pb2.HTTPRequest: return preq -def _extract_http_client(cc: ClientConnection) -> http_pb2.ClientConnection: +def _dump_http_client_conn(cc: ClientConnection) -> http_pb2.ClientConnection: pcc = http_pb2.ClientConnection() _move_attrs(cc, pcc, ['id', 'tls_established', 'timestamp_start', 'timestamp_tls_setup', 'timestamp_end', 'sni', 'cipher_name', 'alpn_proto_negotiated', 'tls_version']) @@ -54,7 +59,7 @@ def _extract_http_client(cc: ClientConnection) -> http_pb2.ClientConnection: return pcc -def _extract_http_server(sc: ServerConnection) -> http_pb2.ServerConnection: +def _dump_http_server_conn(sc: ServerConnection) -> http_pb2.ServerConnection: psc = http_pb2.ServerConnection() _move_attrs(sc, psc, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) @@ -62,14 +67,14 @@ def _extract_http_server(sc: ServerConnection) -> http_pb2.ServerConnection: if hasattr(sc, addr) and getattr(sc, addr) is not None: getattr(psc, addr).host = getattr(sc, addr)[0] getattr(psc, addr).port = getattr(sc, addr)[1] - if psc.cert: + if sc.cert: psc.cert = sc.cert.to_pem() if sc.via: - psc.via.MergeFrom(_extract_http_server(sc.via)) + psc.via.MergeFrom(_dump_http_server_conn(sc.via)) return psc -def _extract_http_error(e: flow.Error) -> http_pb2.HTTPError: +def _dump_http_error(e: flow.Error) -> http_pb2.HTTPError: pe = http_pb2.HTTPError() for attr in ['msg', 'timestamp']: if hasattr(e, attr) and getattr(e, attr) is not None: @@ -77,115 +82,104 @@ def _extract_http_error(e: flow.Error) -> http_pb2.HTTPError: return pe -def extract_http(f: HTTPFlow) -> http_pb2.HTTPFlow(): +def dump_http(f: HTTPFlow) -> http_pb2.HTTPFlow(): pf = http_pb2.HTTPFlow() for p in ['request', 'response', 'client_conn', 'server_conn', 'error']: if hasattr(f, p): - getattr(pf, p).MergeFrom(parsers[p](getattr(f, p))) + getattr(pf, p).MergeFrom(eval(f"_dump_http_{p}")(getattr(f, p))) _move_attrs(f, pf, ['intercepted', 'marked', 'mode', 'id', 'version']) return pf -parsers = { - 'request': _extract_http_request, - 'response': _extract_http_response, - 'error': _extract_http_error, - 'client_conn': _extract_http_client, - 'server_conn': _extract_http_server -} - - def dumps(f: flow.Flow) -> bytes: if f.type != "http": raise exceptions.TypeError("Flow types different than HTTP not supported yet!") else: - p = parse_http(f) + p = dump_http(f) return p.SerializeToString() -def loads(b: bytes, type='http') -> flow.Flow: - if type != 'http': +def _load_http_request(o: http_pb2.HTTPRequest) -> HTTPRequest: + d = {} + _move_attrs(o, d, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content', + 'timestamp_start', 'timestamp_end', 'is_replay']) + d["headers"] = [] + for header in o.headers: + d["headers"].append((bytes(header.name), bytes(header.value))) + + return HTTPRequest(**d) + + +def _load_http_response(o: http_pb2.HTTPResponse) -> HTTPResponse: + d = {} + _move_attrs(o, d, ['http_version', 'status_code', 'reason', + 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) + d["headers"] = [] + for header in o.headers: + d["headers"].append((bytes(header.name), bytes(header.value))) + + return HTTPResponse(**d) + + +def _load_http_client_conn(o: http_pb2.ClientConnection) -> ClientConnection: + d = {} + _move_attrs(o.client_conn, d, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', + 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) + for cert in ['clientcert', 'mitmcert']: + if hasattr(o, cert) and getattr(o, cert) is not None: + c = Cert() + c.from_pem(getattr(o, cert)) + d[cert] = c + if o.tls_extensions: + d['tls_extensions'] = [] + for extension in o.tls_extensions: + d['tls_extensions'].append((extension.int, extension.bytes)) + if o.address: + d['address'] = (o.address.host, o.address.port) + + +def _load_http_server_conn(o: http_pb2.ServerConnection) -> ServerConnection: + d = {} + _move_attrs(o, d, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', + 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) + for addr in ['address', 'ip_address', 'source_address']: + if hasattr(o, addr): + d[addr] = (getattr(o, addr).host, getattr(o, addr).port) + if o.cert: + c = Cert() + c.from_pem(o.cert) + d['cert'] = c + if o.via: + d['via'] = _load_http_server_conn(d['via']) + + return ServerConnection(**d) + + +def _load_http_error(o: http_pb2.HTTPError) -> flow.Error: + d = {} + for m in ['msg', 'timestamp']: + if hasattr(o, m) and getattr(o, m): + d[m] = getattr(o, m) + return flow.Error(**d) + + +def load_http(hf: http_pb2.HTTPFlow) -> HTTPFlow: + parts = {} + for p in ['request', 'response', 'client_conn', 'server_conn', 'error']: + if hasattr(hf, p) and getattr(hf, p): + parts[p] = eval(f"_load_http{p}")(getattr(hf, p)) + _move_attrs(hf, parts, ['intercepted', 'marked', 'mode', 'id', 'version']) + return HTTPFlow(**parts) + + +def loads(b: bytes, typ="http") -> flow.Flow: + if typ != 'http': raise exceptions.TypeError("Flow types different than HTTP not supported yet!") else: p = http_pb2.HTTPFlow() try: p.ParseFromString(b) + return load_http(p) except Exception as e: ctx.log(e.strerror) - -def dumps_state(f: flow.Flow) -> bytes: - if f.type != "http": - raise exceptions.TypeError("Flow types different than HTTP not supported yet!") - else: - hf = http_pb2.HTTPFlow() - state = f.get_state() - for r in ['request', 'response']: - for field in state[r]: - if hasattr(getattr(hf, r), field) and state[r][field] is not None: - if field == 'headers': - for n, v in state[r][field]: - header = (getattr(hf, r)).headers.add() - header.name = n - header.value = v - else: - setattr(getattr(hf, r), field, state[r][field]) - if state['error']: - hf.error.msg = state['error']['msg'] - hf.error.timestamp = state['error']['timestamp'] - for c in ['client_conn', 'server_conn']: - for field in state[c]: - if hasattr(getattr(hf, c), field) and state[c][field] is not None: - if field in ['address', 'ip_address', 'source_address']: - getattr(getattr(hf, c), field).host = state[c][field][0] - getattr(getattr(hf, c), field).port = state[c][field][1] - elif field == "tls_extensions": - for t in state[c][field]: - if t[0] and t[1]: - ext = getattr(hf, c).tls_extensions.add() - ext.int = t[0] - ext.bytes = t[1] - else: - setattr(getattr(hf, c), field, state[c][field]) - for val in ['intercepted', 'marked', 'mode', 'id', 'version']: - if state[val] is not None: - setattr(hf, val, state[val]) - return hf.SerializeToString() - - -def loads_state(blob: bytes) -> flow.Flow: - r = http_pb2.HTTPFlow() - state = dict() - r.ParseFromString(blob) - _dump_object(r, state) - # ugly repair for tls_extensions typing - for c in ["client_conn", "server_conn"]: - state[c]['tls_extensions'] = [] - return HTTPFlow.from_state(state) - - -def _dump_object(obj, d): - for descriptor in obj.DESCRIPTOR.fields: - value = getattr(obj, descriptor.name) - if descriptor.type == descriptor.TYPE_MESSAGE: - if descriptor.label == descriptor.LABEL_REPEATED: - d[descriptor] = [] - if value: - for v in value: - _dump_object(v, d[descriptor.name]) - else: - d[descriptor.name] = None - else: - d[descriptor] = {} - _dump_object(value, d[descriptor.name]) - elif descriptor.type == descriptor.TYPE_ENUM: - enum_name = descriptor.enum_type.values[value].name - d[descriptor.name] = enum_name - else: - if value == "" or value == b"": - d[descriptor.name] = None - else: - if type(d) == list: - d.append([descriptor.name, value]) - elif type(d) == dict: - d[descriptor.name] = value From 189e2ce0d0bd39364c87ce6cb032e21e3c365264 Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 30 Jun 2018 10:58:01 +0200 Subject: [PATCH 10/18] Loads yields now an HTTP object --- mitmproxy/io/protobuf.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py index dd2852869..54dc14377 100644 --- a/mitmproxy/io/protobuf.py +++ b/mitmproxy/io/protobuf.py @@ -1,3 +1,5 @@ +import typing + from mitmproxy import flow from mitmproxy import exceptions from mitmproxy import ctx @@ -105,7 +107,7 @@ def _load_http_request(o: http_pb2.HTTPRequest) -> HTTPRequest: 'timestamp_start', 'timestamp_end', 'is_replay']) d["headers"] = [] for header in o.headers: - d["headers"].append((bytes(header.name), bytes(header.value))) + d["headers"].append((bytes(header.name, "utf-8"), bytes(header.value, "utf-8"))) return HTTPRequest(**d) @@ -116,18 +118,18 @@ def _load_http_response(o: http_pb2.HTTPResponse) -> HTTPResponse: 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) d["headers"] = [] for header in o.headers: - d["headers"].append((bytes(header.name), bytes(header.value))) + d["headers"].append((bytes(header.name, "utf-8"), bytes(header.value, "utf-8"))) return HTTPResponse(**d) def _load_http_client_conn(o: http_pb2.ClientConnection) -> ClientConnection: d = {} - _move_attrs(o.client_conn, d, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', + _move_attrs(o, d, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) for cert in ['clientcert', 'mitmcert']: - if hasattr(o, cert) and getattr(o, cert) is not None: - c = Cert() + if hasattr(o, cert) and getattr(o, cert): + c = Cert("") c.from_pem(getattr(o, cert)) d[cert] = c if o.tls_extensions: @@ -136,6 +138,10 @@ def _load_http_client_conn(o: http_pb2.ClientConnection) -> ClientConnection: d['tls_extensions'].append((extension.int, extension.bytes)) if o.address: d['address'] = (o.address.host, o.address.port) + cc = ClientConnection(None, tuple(), None) + for k, v in d.items(): + setattr(cc, k, v) + return cc def _load_http_server_conn(o: http_pb2.ServerConnection) -> ServerConnection: @@ -146,30 +152,35 @@ def _load_http_server_conn(o: http_pb2.ServerConnection) -> ServerConnection: if hasattr(o, addr): d[addr] = (getattr(o, addr).host, getattr(o, addr).port) if o.cert: - c = Cert() + c = Cert("") c.from_pem(o.cert) d['cert'] = c - if o.via: + if len(o.via.id): d['via'] = _load_http_server_conn(d['via']) - - return ServerConnection(**d) + sc = ServerConnection(tuple()) + for k, v in d.items(): + setattr(sc, k, v) + return sc -def _load_http_error(o: http_pb2.HTTPError) -> flow.Error: +def _load_http_error(o: http_pb2.HTTPError) -> typing.Optional[flow.Error]: d = {} for m in ['msg', 'timestamp']: if hasattr(o, m) and getattr(o, m): d[m] = getattr(o, m) - return flow.Error(**d) + return None if not d else flow.Error(**d) def load_http(hf: http_pb2.HTTPFlow) -> HTTPFlow: parts = {} for p in ['request', 'response', 'client_conn', 'server_conn', 'error']: if hasattr(hf, p) and getattr(hf, p): - parts[p] = eval(f"_load_http{p}")(getattr(hf, p)) + parts[p] = eval(f"_load_http_{p}")(getattr(hf, p)) _move_attrs(hf, parts, ['intercepted', 'marked', 'mode', 'id', 'version']) - return HTTPFlow(**parts) + f = HTTPFlow(ClientConnection(None, tuple(), None), ServerConnection(tuple())) + for k, v in parts.items(): + setattr(f, k, v) + return f def loads(b: bytes, typ="http") -> flow.Flow: @@ -181,5 +192,5 @@ def loads(b: bytes, typ="http") -> flow.Flow: p.ParseFromString(b) return load_http(p) except Exception as e: - ctx.log(e.strerror) + ctx.log(str(e)) From 54cae95a9026e38cc2ab26f3379de11daaa3c399 Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 30 Jun 2018 11:16:59 +0200 Subject: [PATCH 11/18] new protobuf loads has almost the same result of FlowReader stream --- mitmproxy/io/protobuf.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py index 54dc14377..3db9a0132 100644 --- a/mitmproxy/io/protobuf.py +++ b/mitmproxy/io/protobuf.py @@ -16,7 +16,10 @@ def _move_attrs(s_obj, d_obj, attrs): setattr(d_obj, attr, getattr(s_obj, attr)) else: if hasattr(s_obj, attr) and getattr(s_obj, attr) is not None: - d_obj[attr] = getattr(s_obj, attr) + if not getattr(s_obj, attr): + d_obj[attr] = None + else: + d_obj[attr] = getattr(s_obj, attr) def _dump_http_response(res: HTTPResponse) -> http_pb2.HTTPResponse: @@ -105,6 +108,8 @@ def _load_http_request(o: http_pb2.HTTPRequest) -> HTTPRequest: d = {} _move_attrs(o, d, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) + if d['content'] is None: + d['content'] = b"" d["headers"] = [] for header in o.headers: d["headers"].append((bytes(header.name, "utf-8"), bytes(header.value, "utf-8"))) @@ -116,6 +121,8 @@ def _load_http_response(o: http_pb2.HTTPResponse) -> HTTPResponse: d = {} _move_attrs(o, d, ['http_version', 'status_code', 'reason', 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) + if d['content'] is None: + d['content'] = b"" d["headers"] = [] for header in o.headers: d["headers"].append((bytes(header.name, "utf-8"), bytes(header.value, "utf-8"))) From 9ee8f47cc67a556d9e51741936c918b0d1e3a519 Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 30 Jun 2018 11:30:59 +0200 Subject: [PATCH 12/18] bug fix --- mitmproxy/io/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mitmproxy/io/db.py b/mitmproxy/io/db.py index 63cb0880a..5fded6201 100644 --- a/mitmproxy/io/db.py +++ b/mitmproxy/io/db.py @@ -40,3 +40,4 @@ class DbHandler: self._c.execute('SELECT pbuf_blob FROM FLOWS') for row in self._c.fetchall(): flows.append((protobuf.loads(row[0]))) + return flows From 049b096cdab158e9266078a7fa03fbfef2b2fb88 Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 30 Jun 2018 11:42:16 +0200 Subject: [PATCH 13/18] more bug fixes in DB class --- mitmproxy/io/db.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mitmproxy/io/db.py b/mitmproxy/io/db.py index 5fded6201..01d30aa29 100644 --- a/mitmproxy/io/db.py +++ b/mitmproxy/io/db.py @@ -13,10 +13,6 @@ class DbHandler: """ def __init__(self, db_path="/tmp/tmp.sqlite"): - if os.path.isfile(db_path): - self.db_path = db_path - else: - raise IOError("Invalid path!") self.db_path = db_path self._con = sqlite3.connect(self.db_path) self._c = self._con.cursor() From 85da4694f111f1029bad878000b1d40b49aceb7d Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 30 Jun 2018 11:48:06 +0200 Subject: [PATCH 14/18] Timestamps are now double instead of float --- mitmproxy/io/proto/http.proto | 24 ++++++++++++------------ mitmproxy/io/proto/http_pb2.py | 26 +++++++++++++------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/mitmproxy/io/proto/http.proto b/mitmproxy/io/proto/http.proto index 30c11a0a7..6c3724334 100644 --- a/mitmproxy/io/proto/http.proto +++ b/mitmproxy/io/proto/http.proto @@ -23,8 +23,8 @@ message HTTPRequest { optional string http_version = 7; repeated HTTPHeader headers = 8; optional bytes content = 9; - optional float timestamp_start = 10; - optional float timestamp_end = 11; + optional double timestamp_start = 10; + optional double timestamp_end = 11; optional bool is_replay = 12; } @@ -34,14 +34,14 @@ message HTTPResponse { optional string reason = 3; repeated HTTPHeader headers = 4; optional bytes content = 5; - optional float timestamp_start = 6; - optional float timestamp_end = 7; + optional double timestamp_start = 6; + optional double timestamp_end = 7; optional bool is_replay = 8; } message HTTPError { optional string msg = 1; - optional float timestamp = 2; + optional double timestamp = 2; } message HTTPHeader { @@ -61,9 +61,9 @@ message ClientConnection { optional bool tls_established = 3; optional string clientcert = 4; optional string mitmcert = 5; - optional float timestamp_start = 6; - optional float timestamp_tls_setup = 7; - optional float timestamp_end = 8; + optional double timestamp_start = 6; + optional double timestamp_tls_setup = 7; + optional double timestamp_end = 8; optional string sni = 9; optional string cipher_name = 10; optional bytes alpn_proto_negotiated = 11; @@ -81,10 +81,10 @@ message ServerConnection { optional string sni = 7; optional bytes alpn_proto_negotiated = 8; optional string tls_version = 9; - optional float timestamp_start = 10; - optional float timestamp_tcp_setup = 11; - optional float timestamp_tls_setup = 12; - optional float timestamp_end = 13; + optional double timestamp_start = 10; + optional double timestamp_tcp_setup = 11; + optional double timestamp_tls_setup = 12; + optional double timestamp_end = 13; optional ServerConnection via = 14; } diff --git a/mitmproxy/io/proto/http_pb2.py b/mitmproxy/io/proto/http_pb2.py index d766a4ccb..9809e7ee9 100644 --- a/mitmproxy/io/proto/http_pb2.py +++ b/mitmproxy/io/proto/http_pb2.py @@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='http.proto', package='', syntax='proto2', - serialized_pb=_b('\n\nhttp.proto\"\x85\x02\n\x08HTTPFlow\x12\x1d\n\x07request\x18\x01 \x01(\x0b\x32\x0c.HTTPRequest\x12\x1f\n\x08response\x18\x02 \x01(\x0b\x32\r.HTTPResponse\x12\x19\n\x05\x65rror\x18\x03 \x01(\x0b\x32\n.HTTPError\x12&\n\x0b\x63lient_conn\x18\x04 \x01(\x0b\x32\x11.ClientConnection\x12&\n\x0bserver_conn\x18\x05 \x01(\x0b\x32\x11.ServerConnection\x12\x13\n\x0bintercepted\x18\x06 \x01(\x08\x12\x0e\n\x06marked\x18\x07 \x01(\x08\x12\x0c\n\x04mode\x18\x08 \x01(\t\x12\n\n\x02id\x18\t \x01(\t\x12\x0f\n\x07version\x18\n \x01(\x05\"\xfa\x01\n\x0bHTTPRequest\x12\x19\n\x11\x66irst_line_format\x18\x01 \x01(\t\x12\x0e\n\x06method\x18\x02 \x01(\t\x12\x0e\n\x06scheme\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\x05\x12\x0c\n\x04path\x18\x06 \x01(\t\x12\x14\n\x0chttp_version\x18\x07 \x01(\t\x12\x1c\n\x07headers\x18\x08 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\t \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x15\n\rtimestamp_end\x18\x0b \x01(\x02\x12\x11\n\tis_replay\x18\x0c \x01(\x08\"\xbb\x01\n\x0cHTTPResponse\x12\x14\n\x0chttp_version\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x1c\n\x07headers\x18\x04 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x07 \x01(\x02\x12\x11\n\tis_replay\x18\x08 \x01(\x08\"+\n\tHTTPError\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x02\")\n\nHTTPHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"%\n\x07\x41\x64\x64ress\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xc2\x02\n\x10\x43lientConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x03 \x01(\x08\x12\x12\n\nclientcert\x18\x04 \x01(\t\x12\x10\n\x08mitmcert\x18\x05 \x01(\t\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x07 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x08 \x01(\x02\x12\x0b\n\x03sni\x18\t \x01(\t\x12\x13\n\x0b\x63ipher_name\x18\n \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x0b \x01(\x0c\x12\x13\n\x0btls_version\x18\x0c \x01(\t\x12%\n\x0etls_extensions\x18\r \x03(\x0b\x32\r.TLSExtension\"\xeb\x02\n\x10ServerConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x1c\n\nip_address\x18\x03 \x01(\x0b\x32\x08.Address\x12 \n\x0esource_address\x18\x04 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x05 \x01(\x08\x12\x0c\n\x04\x63\x65rt\x18\x06 \x01(\t\x12\x0b\n\x03sni\x18\x07 \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x08 \x01(\x0c\x12\x13\n\x0btls_version\x18\t \x01(\t\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x1b\n\x13timestamp_tcp_setup\x18\x0b \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x0c \x01(\x02\x12\x15\n\rtimestamp_end\x18\r \x01(\x02\x12\x1e\n\x03via\x18\x0e \x01(\x0b\x32\x11.ServerConnection\"*\n\x0cTLSExtension\x12\x0b\n\x03int\x18\x01 \x01(\x03\x12\r\n\x05\x62ytes\x18\x02 \x01(\x0c') + serialized_pb=_b('\n\nhttp.proto\"\x85\x02\n\x08HTTPFlow\x12\x1d\n\x07request\x18\x01 \x01(\x0b\x32\x0c.HTTPRequest\x12\x1f\n\x08response\x18\x02 \x01(\x0b\x32\r.HTTPResponse\x12\x19\n\x05\x65rror\x18\x03 \x01(\x0b\x32\n.HTTPError\x12&\n\x0b\x63lient_conn\x18\x04 \x01(\x0b\x32\x11.ClientConnection\x12&\n\x0bserver_conn\x18\x05 \x01(\x0b\x32\x11.ServerConnection\x12\x13\n\x0bintercepted\x18\x06 \x01(\x08\x12\x0e\n\x06marked\x18\x07 \x01(\x08\x12\x0c\n\x04mode\x18\x08 \x01(\t\x12\n\n\x02id\x18\t \x01(\t\x12\x0f\n\x07version\x18\n \x01(\x05\"\xfa\x01\n\x0bHTTPRequest\x12\x19\n\x11\x66irst_line_format\x18\x01 \x01(\t\x12\x0e\n\x06method\x18\x02 \x01(\t\x12\x0e\n\x06scheme\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\x05\x12\x0c\n\x04path\x18\x06 \x01(\t\x12\x14\n\x0chttp_version\x18\x07 \x01(\t\x12\x1c\n\x07headers\x18\x08 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\t \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x01\x12\x15\n\rtimestamp_end\x18\x0b \x01(\x01\x12\x11\n\tis_replay\x18\x0c \x01(\x08\"\xbb\x01\n\x0cHTTPResponse\x12\x14\n\x0chttp_version\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x1c\n\x07headers\x18\x04 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x01\x12\x15\n\rtimestamp_end\x18\x07 \x01(\x01\x12\x11\n\tis_replay\x18\x08 \x01(\x08\"+\n\tHTTPError\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x01\")\n\nHTTPHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"%\n\x07\x41\x64\x64ress\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xc2\x02\n\x10\x43lientConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x03 \x01(\x08\x12\x12\n\nclientcert\x18\x04 \x01(\t\x12\x10\n\x08mitmcert\x18\x05 \x01(\t\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x01\x12\x1b\n\x13timestamp_tls_setup\x18\x07 \x01(\x01\x12\x15\n\rtimestamp_end\x18\x08 \x01(\x01\x12\x0b\n\x03sni\x18\t \x01(\t\x12\x13\n\x0b\x63ipher_name\x18\n \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x0b \x01(\x0c\x12\x13\n\x0btls_version\x18\x0c \x01(\t\x12%\n\x0etls_extensions\x18\r \x03(\x0b\x32\r.TLSExtension\"\xeb\x02\n\x10ServerConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x1c\n\nip_address\x18\x03 \x01(\x0b\x32\x08.Address\x12 \n\x0esource_address\x18\x04 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x05 \x01(\x08\x12\x0c\n\x04\x63\x65rt\x18\x06 \x01(\t\x12\x0b\n\x03sni\x18\x07 \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x08 \x01(\x0c\x12\x13\n\x0btls_version\x18\t \x01(\t\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x01\x12\x1b\n\x13timestamp_tcp_setup\x18\x0b \x01(\x01\x12\x1b\n\x13timestamp_tls_setup\x18\x0c \x01(\x01\x12\x15\n\rtimestamp_end\x18\r \x01(\x01\x12\x1e\n\x03via\x18\x0e \x01(\x0b\x32\x11.ServerConnection\"*\n\x0cTLSExtension\x12\x0b\n\x03int\x18\x01 \x01(\x03\x12\r\n\x05\x62ytes\x18\x02 \x01(\x0c') ) @@ -191,14 +191,14 @@ _HTTPREQUEST = _descriptor.Descriptor( options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp_start', full_name='HTTPRequest.timestamp_start', index=9, - number=10, type=2, cpp_type=6, label=1, + number=10, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp_end', full_name='HTTPRequest.timestamp_end', index=10, - number=11, type=2, cpp_type=6, label=1, + number=11, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -271,14 +271,14 @@ _HTTPRESPONSE = _descriptor.Descriptor( options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp_start', full_name='HTTPResponse.timestamp_start', index=5, - number=6, type=2, cpp_type=6, label=1, + number=6, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp_end', full_name='HTTPResponse.timestamp_end', index=6, - number=7, type=2, cpp_type=6, label=1, + number=7, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -323,7 +323,7 @@ _HTTPERROR = _descriptor.Descriptor( options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp', full_name='HTTPError.timestamp', index=1, - number=2, type=2, cpp_type=6, label=1, + number=2, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -465,21 +465,21 @@ _CLIENTCONNECTION = _descriptor.Descriptor( options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp_start', full_name='ClientConnection.timestamp_start', index=5, - number=6, type=2, cpp_type=6, label=1, + number=6, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp_tls_setup', full_name='ClientConnection.timestamp_tls_setup', index=6, - number=7, type=2, cpp_type=6, label=1, + number=7, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp_end', full_name='ClientConnection.timestamp_end', index=7, - number=8, type=2, cpp_type=6, label=1, + number=8, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -608,28 +608,28 @@ _SERVERCONNECTION = _descriptor.Descriptor( options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp_start', full_name='ServerConnection.timestamp_start', index=9, - number=10, type=2, cpp_type=6, label=1, + number=10, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp_tcp_setup', full_name='ServerConnection.timestamp_tcp_setup', index=10, - number=11, type=2, cpp_type=6, label=1, + number=11, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp_tls_setup', full_name='ServerConnection.timestamp_tls_setup', index=11, - number=12, type=2, cpp_type=6, label=1, + number=12, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='timestamp_end', full_name='ServerConnection.timestamp_end', index=12, - number=13, type=2, cpp_type=6, label=1, + number=13, type=1, cpp_type=5, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, From 66aff8d2a46f800348408a247a057b4fc8a883b5 Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 30 Jun 2018 12:20:27 +0200 Subject: [PATCH 15/18] view can load from both io interfaces --- mitmproxy/addons/view.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 3730aa8d6..25142a2c2 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -447,9 +447,17 @@ class View(collections.Sequence): Load flows into the view, without processing them with addons. """ try: - dh = io.DbHandler(path) - for f in dh.load(): - self.add([f.copy()]) + if path.endswith(".sqlite"): + dh = io.DbHandler(path) + for f in dh.load(): + self.add([f.copy()]) + else: + with open(path, "rb") as f: + for i in io.FlowReader(f).stream(): + # Do this to get a new ID, so we can load the same file N times and + # get new flows each time. It would be more efficient to just have a + # .newid() method or something. + self.add([i.copy()]) except exceptions.TypeError as e: ctx.log.error(str(e)) except IOError as e: From cbb6e381b96e74d34de08e152d864b62b55b0176 Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 30 Jun 2018 12:23:40 +0200 Subject: [PATCH 16/18] updated exceptions in view load --- mitmproxy/addons/view.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 25142a2c2..57aa4b315 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -462,6 +462,8 @@ class View(collections.Sequence): ctx.log.error(str(e)) except IOError as e: ctx.log.error(e.strerror) + except exceptions.FlowReadException as e: + ctx.log.error(str(e)) def add(self, flows: typing.Sequence[mitmproxy.flow.Flow]) -> None: """ From 204faa1f524cdd5dcd997237b017ffa045aad1ec Mon Sep 17 00:00:00 2001 From: Pietro Francesco Tirenna Date: Sat, 30 Jun 2018 15:52:48 +0200 Subject: [PATCH 17/18] Removed test code from view --- mitmproxy/addons/view.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 57aa4b315..1c8bd0cee 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -447,19 +447,12 @@ class View(collections.Sequence): Load flows into the view, without processing them with addons. """ try: - if path.endswith(".sqlite"): - dh = io.DbHandler(path) - for f in dh.load(): - self.add([f.copy()]) - else: - with open(path, "rb") as f: - for i in io.FlowReader(f).stream(): - # Do this to get a new ID, so we can load the same file N times and - # get new flows each time. It would be more efficient to just have a - # .newid() method or something. - self.add([i.copy()]) - except exceptions.TypeError as e: - ctx.log.error(str(e)) + with open(path, "rb") as f: + for i in io.FlowReader(f).stream(): + # Do this to get a new ID, so we can load the same file N times and + # get new flows each time. It would be more efficient to just have a + # .newid() method or something. + self.add([i.copy()]) except IOError as e: ctx.log.error(e.strerror) except exceptions.FlowReadException as e: From d5da74645b1231783eb5d527f226d0b06ffd2c4d Mon Sep 17 00:00:00 2001 From: Pietro Francesco Tirenna Date: Mon, 2 Jul 2018 15:59:48 +0200 Subject: [PATCH 18/18] protobuf: tests implemented, full coverage --- .gitignore | 1 + mitmproxy/io/__init__.py | 4 +- mitmproxy/io/db.py | 9 ++- mitmproxy/io/proto/http.proto | 1 - mitmproxy/io/proto/http_pb2.py | 43 +++++------ mitmproxy/io/protobuf.py | 53 ++++++------- setup.cfg | 2 +- setup.py | 2 +- test/filename_matching.py | 3 +- test/mitmproxy/io/test_db.py | 26 +++++++ test/mitmproxy/io/test_protobuf.py | 120 +++++++++++++++++++++++++++++ 11 files changed, 200 insertions(+), 64 deletions(-) create mode 100644 test/mitmproxy/io/test_db.py create mode 100644 test/mitmproxy/io/test_protobuf.py diff --git a/.gitignore b/.gitignore index 2ada574a3..deb938149 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ MANIFEST *.py[cdo] *.swp *.swo +*.sqlite *.egg-info/ .coverage* .idea diff --git a/mitmproxy/io/__init__.py b/mitmproxy/io/__init__.py index 854d25058..bd248fd2e 100644 --- a/mitmproxy/io/__init__.py +++ b/mitmproxy/io/__init__.py @@ -1,8 +1,8 @@ from .io import FlowWriter, FlowReader, FilteredFlowWriter, read_flows_from_paths -from .db import DbHandler +from .db import DBHandler __all__ = [ - "FlowWriter", "FlowReader", "FilteredFlowWriter", "read_flows_from_paths", "DbHandler" + "FlowWriter", "FlowReader", "FilteredFlowWriter", "read_flows_from_paths", "DBHandler" ] diff --git a/mitmproxy/io/db.py b/mitmproxy/io/db.py index 01d30aa29..ea42e08b0 100644 --- a/mitmproxy/io/db.py +++ b/mitmproxy/io/db.py @@ -2,17 +2,18 @@ import sqlite3 import os from mitmproxy.io import protobuf -from mitmproxy.http import HTTPFlow -from mitmproxy import exceptions -class DbHandler: +class DBHandler: """ This class is wrapping up connection to SQLITE DB. """ - def __init__(self, db_path="/tmp/tmp.sqlite"): + def __init__(self, db_path, mode='load'): + if mode == 'write': + if os.path.isfile(db_path): + os.remove(db_path) self.db_path = db_path self._con = sqlite3.connect(self.db_path) self._c = self._con.cursor() diff --git a/mitmproxy/io/proto/http.proto b/mitmproxy/io/proto/http.proto index 6c3724334..c86a04f36 100644 --- a/mitmproxy/io/proto/http.proto +++ b/mitmproxy/io/proto/http.proto @@ -10,7 +10,6 @@ message HTTPFlow { optional bool marked = 7; optional string mode = 8; optional string id = 9; - optional int32 version = 10; } message HTTPRequest { diff --git a/mitmproxy/io/proto/http_pb2.py b/mitmproxy/io/proto/http_pb2.py index 9809e7ee9..47b09815b 100644 --- a/mitmproxy/io/proto/http_pb2.py +++ b/mitmproxy/io/proto/http_pb2.py @@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='http.proto', package='', syntax='proto2', - serialized_pb=_b('\n\nhttp.proto\"\x85\x02\n\x08HTTPFlow\x12\x1d\n\x07request\x18\x01 \x01(\x0b\x32\x0c.HTTPRequest\x12\x1f\n\x08response\x18\x02 \x01(\x0b\x32\r.HTTPResponse\x12\x19\n\x05\x65rror\x18\x03 \x01(\x0b\x32\n.HTTPError\x12&\n\x0b\x63lient_conn\x18\x04 \x01(\x0b\x32\x11.ClientConnection\x12&\n\x0bserver_conn\x18\x05 \x01(\x0b\x32\x11.ServerConnection\x12\x13\n\x0bintercepted\x18\x06 \x01(\x08\x12\x0e\n\x06marked\x18\x07 \x01(\x08\x12\x0c\n\x04mode\x18\x08 \x01(\t\x12\n\n\x02id\x18\t \x01(\t\x12\x0f\n\x07version\x18\n \x01(\x05\"\xfa\x01\n\x0bHTTPRequest\x12\x19\n\x11\x66irst_line_format\x18\x01 \x01(\t\x12\x0e\n\x06method\x18\x02 \x01(\t\x12\x0e\n\x06scheme\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\x05\x12\x0c\n\x04path\x18\x06 \x01(\t\x12\x14\n\x0chttp_version\x18\x07 \x01(\t\x12\x1c\n\x07headers\x18\x08 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\t \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x01\x12\x15\n\rtimestamp_end\x18\x0b \x01(\x01\x12\x11\n\tis_replay\x18\x0c \x01(\x08\"\xbb\x01\n\x0cHTTPResponse\x12\x14\n\x0chttp_version\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x1c\n\x07headers\x18\x04 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x01\x12\x15\n\rtimestamp_end\x18\x07 \x01(\x01\x12\x11\n\tis_replay\x18\x08 \x01(\x08\"+\n\tHTTPError\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x01\")\n\nHTTPHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"%\n\x07\x41\x64\x64ress\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xc2\x02\n\x10\x43lientConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x03 \x01(\x08\x12\x12\n\nclientcert\x18\x04 \x01(\t\x12\x10\n\x08mitmcert\x18\x05 \x01(\t\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x01\x12\x1b\n\x13timestamp_tls_setup\x18\x07 \x01(\x01\x12\x15\n\rtimestamp_end\x18\x08 \x01(\x01\x12\x0b\n\x03sni\x18\t \x01(\t\x12\x13\n\x0b\x63ipher_name\x18\n \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x0b \x01(\x0c\x12\x13\n\x0btls_version\x18\x0c \x01(\t\x12%\n\x0etls_extensions\x18\r \x03(\x0b\x32\r.TLSExtension\"\xeb\x02\n\x10ServerConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x1c\n\nip_address\x18\x03 \x01(\x0b\x32\x08.Address\x12 \n\x0esource_address\x18\x04 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x05 \x01(\x08\x12\x0c\n\x04\x63\x65rt\x18\x06 \x01(\t\x12\x0b\n\x03sni\x18\x07 \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x08 \x01(\x0c\x12\x13\n\x0btls_version\x18\t \x01(\t\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x01\x12\x1b\n\x13timestamp_tcp_setup\x18\x0b \x01(\x01\x12\x1b\n\x13timestamp_tls_setup\x18\x0c \x01(\x01\x12\x15\n\rtimestamp_end\x18\r \x01(\x01\x12\x1e\n\x03via\x18\x0e \x01(\x0b\x32\x11.ServerConnection\"*\n\x0cTLSExtension\x12\x0b\n\x03int\x18\x01 \x01(\x03\x12\r\n\x05\x62ytes\x18\x02 \x01(\x0c') + serialized_pb=_b('\n\nhttp.proto\"\xf4\x01\n\x08HTTPFlow\x12\x1d\n\x07request\x18\x01 \x01(\x0b\x32\x0c.HTTPRequest\x12\x1f\n\x08response\x18\x02 \x01(\x0b\x32\r.HTTPResponse\x12\x19\n\x05\x65rror\x18\x03 \x01(\x0b\x32\n.HTTPError\x12&\n\x0b\x63lient_conn\x18\x04 \x01(\x0b\x32\x11.ClientConnection\x12&\n\x0bserver_conn\x18\x05 \x01(\x0b\x32\x11.ServerConnection\x12\x13\n\x0bintercepted\x18\x06 \x01(\x08\x12\x0e\n\x06marked\x18\x07 \x01(\x08\x12\x0c\n\x04mode\x18\x08 \x01(\t\x12\n\n\x02id\x18\t \x01(\t\"\xfa\x01\n\x0bHTTPRequest\x12\x19\n\x11\x66irst_line_format\x18\x01 \x01(\t\x12\x0e\n\x06method\x18\x02 \x01(\t\x12\x0e\n\x06scheme\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\x05\x12\x0c\n\x04path\x18\x06 \x01(\t\x12\x14\n\x0chttp_version\x18\x07 \x01(\t\x12\x1c\n\x07headers\x18\x08 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\t \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x01\x12\x15\n\rtimestamp_end\x18\x0b \x01(\x01\x12\x11\n\tis_replay\x18\x0c \x01(\x08\"\xbb\x01\n\x0cHTTPResponse\x12\x14\n\x0chttp_version\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x1c\n\x07headers\x18\x04 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x01\x12\x15\n\rtimestamp_end\x18\x07 \x01(\x01\x12\x11\n\tis_replay\x18\x08 \x01(\x08\"+\n\tHTTPError\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x01\")\n\nHTTPHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"%\n\x07\x41\x64\x64ress\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xc2\x02\n\x10\x43lientConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x03 \x01(\x08\x12\x12\n\nclientcert\x18\x04 \x01(\t\x12\x10\n\x08mitmcert\x18\x05 \x01(\t\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x01\x12\x1b\n\x13timestamp_tls_setup\x18\x07 \x01(\x01\x12\x15\n\rtimestamp_end\x18\x08 \x01(\x01\x12\x0b\n\x03sni\x18\t \x01(\t\x12\x13\n\x0b\x63ipher_name\x18\n \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x0b \x01(\x0c\x12\x13\n\x0btls_version\x18\x0c \x01(\t\x12%\n\x0etls_extensions\x18\r \x03(\x0b\x32\r.TLSExtension\"\xeb\x02\n\x10ServerConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x1c\n\nip_address\x18\x03 \x01(\x0b\x32\x08.Address\x12 \n\x0esource_address\x18\x04 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x05 \x01(\x08\x12\x0c\n\x04\x63\x65rt\x18\x06 \x01(\t\x12\x0b\n\x03sni\x18\x07 \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x08 \x01(\x0c\x12\x13\n\x0btls_version\x18\t \x01(\t\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x01\x12\x1b\n\x13timestamp_tcp_setup\x18\x0b \x01(\x01\x12\x1b\n\x13timestamp_tls_setup\x18\x0c \x01(\x01\x12\x15\n\rtimestamp_end\x18\r \x01(\x01\x12\x1e\n\x03via\x18\x0e \x01(\x0b\x32\x11.ServerConnection\"*\n\x0cTLSExtension\x12\x0b\n\x03int\x18\x01 \x01(\x03\x12\r\n\x05\x62ytes\x18\x02 \x01(\x0c') ) @@ -95,13 +95,6 @@ _HTTPFLOW = _descriptor.Descriptor( message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='version', full_name='HTTPFlow.version', index=9, - number=10, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -115,7 +108,7 @@ _HTTPFLOW = _descriptor.Descriptor( oneofs=[ ], serialized_start=15, - serialized_end=276, + serialized_end=259, ) @@ -222,8 +215,8 @@ _HTTPREQUEST = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=279, - serialized_end=529, + serialized_start=262, + serialized_end=512, ) @@ -302,8 +295,8 @@ _HTTPRESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=532, - serialized_end=719, + serialized_start=515, + serialized_end=702, ) @@ -340,8 +333,8 @@ _HTTPERROR = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=721, - serialized_end=764, + serialized_start=704, + serialized_end=747, ) @@ -378,8 +371,8 @@ _HTTPHEADER = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=766, - serialized_end=807, + serialized_start=749, + serialized_end=790, ) @@ -416,8 +409,8 @@ _ADDRESS = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=809, - serialized_end=846, + serialized_start=792, + serialized_end=829, ) @@ -531,8 +524,8 @@ _CLIENTCONNECTION = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=849, - serialized_end=1171, + serialized_start=832, + serialized_end=1154, ) @@ -653,8 +646,8 @@ _SERVERCONNECTION = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1174, - serialized_end=1537, + serialized_start=1157, + serialized_end=1520, ) @@ -691,8 +684,8 @@ _TLSEXTENSION = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=1539, - serialized_end=1581, + serialized_start=1522, + serialized_end=1564, ) _HTTPFLOW.fields_by_name['request'].message_type = _HTTPREQUEST diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py index 3db9a0132..9a00eacf7 100644 --- a/mitmproxy/io/protobuf.py +++ b/mitmproxy/io/protobuf.py @@ -2,7 +2,6 @@ import typing from mitmproxy import flow from mitmproxy import exceptions -from mitmproxy import ctx from mitmproxy.http import HTTPFlow, HTTPResponse, HTTPRequest from mitmproxy.certs import Cert from mitmproxy.connections import ClientConnection, ServerConnection @@ -16,7 +15,8 @@ def _move_attrs(s_obj, d_obj, attrs): setattr(d_obj, attr, getattr(s_obj, attr)) else: if hasattr(s_obj, attr) and getattr(s_obj, attr) is not None: - if not getattr(s_obj, attr): + # ugly fix to set None in empty str or bytes fields + if getattr(s_obj, attr) == "" or getattr(s_obj, attr) == b"": d_obj[attr] = None else: d_obj[attr] = getattr(s_obj, attr) @@ -87,12 +87,12 @@ def _dump_http_error(e: flow.Error) -> http_pb2.HTTPError: return pe -def dump_http(f: HTTPFlow) -> http_pb2.HTTPFlow(): +def dump_http(f: flow.Flow) -> http_pb2.HTTPFlow: pf = http_pb2.HTTPFlow() for p in ['request', 'response', 'client_conn', 'server_conn', 'error']: - if hasattr(f, p): + if hasattr(f, p) and getattr(f, p): getattr(pf, p).MergeFrom(eval(f"_dump_http_{p}")(getattr(f, p))) - _move_attrs(f, pf, ['intercepted', 'marked', 'mode', 'id', 'version']) + _move_attrs(f, pf, ['intercepted', 'marked', 'mode', 'id']) return pf @@ -105,9 +105,9 @@ def dumps(f: flow.Flow) -> bytes: def _load_http_request(o: http_pb2.HTTPRequest) -> HTTPRequest: - d = {} + d: dict = {} _move_attrs(o, d, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content', - 'timestamp_start', 'timestamp_end', 'is_replay']) + 'timestamp_start', 'timestamp_end', 'is_replay']) if d['content'] is None: d['content'] = b"" d["headers"] = [] @@ -118,9 +118,9 @@ def _load_http_request(o: http_pb2.HTTPRequest) -> HTTPRequest: def _load_http_response(o: http_pb2.HTTPResponse) -> HTTPResponse: - d = {} + d: dict = {} _move_attrs(o, d, ['http_version', 'status_code', 'reason', - 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) + 'content', 'timestamp_start', 'timestamp_end', 'is_replay']) if d['content'] is None: d['content'] = b"" d["headers"] = [] @@ -131,14 +131,12 @@ def _load_http_response(o: http_pb2.HTTPResponse) -> HTTPResponse: def _load_http_client_conn(o: http_pb2.ClientConnection) -> ClientConnection: - d = {} - _move_attrs(o, d, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', - 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) + d: dict = {} + _move_attrs(o, d, ['id', 'tls_established', 'sni', 'cipher_name', 'alpn_proto_negotiated', 'tls_version', + 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) for cert in ['clientcert', 'mitmcert']: if hasattr(o, cert) and getattr(o, cert): - c = Cert("") - c.from_pem(getattr(o, cert)) - d[cert] = c + d[cert] = Cert.from_pem(getattr(o, cert)) if o.tls_extensions: d['tls_extensions'] = [] for extension in o.tls_extensions: @@ -152,18 +150,17 @@ def _load_http_client_conn(o: http_pb2.ClientConnection) -> ClientConnection: def _load_http_server_conn(o: http_pb2.ServerConnection) -> ServerConnection: - d = {} + d: dict = {} _move_attrs(o, d, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version', - 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) + 'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end']) for addr in ['address', 'ip_address', 'source_address']: if hasattr(o, addr): d[addr] = (getattr(o, addr).host, getattr(o, addr).port) if o.cert: - c = Cert("") - c.from_pem(o.cert) + c = Cert.from_pem(o.cert) d['cert'] = c - if len(o.via.id): - d['via'] = _load_http_server_conn(d['via']) + if o.HasField('via'): + d['via'] = _load_http_server_conn(o.via) sc = ServerConnection(tuple()) for k, v in d.items(): setattr(sc, k, v) @@ -181,9 +178,11 @@ def _load_http_error(o: http_pb2.HTTPError) -> typing.Optional[flow.Error]: def load_http(hf: http_pb2.HTTPFlow) -> HTTPFlow: parts = {} for p in ['request', 'response', 'client_conn', 'server_conn', 'error']: - if hasattr(hf, p) and getattr(hf, p): + if hf.HasField(p): parts[p] = eval(f"_load_http_{p}")(getattr(hf, p)) - _move_attrs(hf, parts, ['intercepted', 'marked', 'mode', 'id', 'version']) + else: + parts[p] = None + _move_attrs(hf, parts, ['intercepted', 'marked', 'mode', 'id']) f = HTTPFlow(ClientConnection(None, tuple(), None), ServerConnection(tuple())) for k, v in parts.items(): setattr(f, k, v) @@ -195,9 +194,5 @@ def loads(b: bytes, typ="http") -> flow.Flow: raise exceptions.TypeError("Flow types different than HTTP not supported yet!") else: p = http_pb2.HTTPFlow() - try: - p.ParseFromString(b) - return load_http(p) - except Exception as e: - ctx.log(str(e)) - + p.ParseFromString(b) + return load_http(p) diff --git a/setup.cfg b/setup.cfg index 32b6aac3c..173166d13 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ max-line-length = 140 max-complexity = 25 ignore = E251,C901,W503,W292,E722,E741 -exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/* +exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/*,mitmproxy/io/proto/* addons = file,open,basestring,xrange,unicode,long,cmp [tool:pytest] diff --git a/setup.py b/setup.py index acd3f2952..954327e9b 100644 --- a/setup.py +++ b/setup.py @@ -70,11 +70,11 @@ setup( "kaitaistruct>=0.7,<0.9", "ldap3>=2.5,<2.6", "passlib>=1.6.5, <1.8", + "protobuf>=3.6.0, <3.7", "pyasn1>=0.3.1,<0.5", "pyOpenSSL>=17.5,<18.1", "pyparsing>=2.1.3, <2.3", "pyperclip>=1.6.0, <1.7", - "protobuf>=3.6.0, <3.7", "ruamel.yaml>=0.13.2, <0.16", "sortedcontainers>=1.5.4,<2.1", "tornado>=4.3,<5.1", diff --git a/test/filename_matching.py b/test/filename_matching.py index 5f49725e6..f53213072 100755 --- a/test/filename_matching.py +++ b/test/filename_matching.py @@ -9,7 +9,8 @@ import sys def check_src_files_have_test(): missing_test_files = [] - excluded = ['mitmproxy/contrib/', 'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/'] + excluded = ['mitmproxy/contrib/', 'mitmproxy/io/proto/', + 'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/'] src_files = glob.glob('mitmproxy/**/*.py', recursive=True) + glob.glob('pathod/**/*.py', recursive=True) src_files = [f for f in src_files if os.path.basename(f) != '__init__.py'] src_files = [f for f in src_files if not any(os.path.normpath(p) in f for p in excluded)] diff --git a/test/mitmproxy/io/test_db.py b/test/mitmproxy/io/test_db.py new file mode 100644 index 000000000..4a2dfb671 --- /dev/null +++ b/test/mitmproxy/io/test_db.py @@ -0,0 +1,26 @@ +from mitmproxy.io import db +from mitmproxy.test import tflow + + +class TestDB: + + def test_create(self, tdata): + dh = db.DBHandler(db_path=tdata.path("mitmproxy/data") + "/tmp.sqlite") + with dh._con as c: + cur = c.cursor() + cur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='FLOWS';") + assert cur.fetchall() == [('FLOWS',)] + + def test_roundtrip(self, tdata): + dh = db.DBHandler(db_path=tdata.path("mitmproxy/data") + "/tmp.sqlite", mode='write') + flows = [] + for i in range(10): + flows.append(tflow.tflow()) + dh.store(flows) + dh = db.DBHandler(db_path=tdata.path("mitmproxy/data") + "/tmp.sqlite") + with dh._con as c: + cur = c.cursor() + cur.execute("SELECT count(*) FROM FLOWS;") + assert cur.fetchall()[0][0] == 10 + loaded_flows = dh.load() + assert len(loaded_flows) == len(flows) diff --git a/test/mitmproxy/io/test_protobuf.py b/test/mitmproxy/io/test_protobuf.py new file mode 100644 index 000000000..f725b9809 --- /dev/null +++ b/test/mitmproxy/io/test_protobuf.py @@ -0,0 +1,120 @@ +import pytest + +from mitmproxy import certs +from mitmproxy import http +from mitmproxy import exceptions +from mitmproxy.test import tflow, tutils +from mitmproxy.io import protobuf + + +class TestProtobuf: + + def test_roundtrip_client(self): + c = tflow.tclient_conn() + del c.reply + c.rfile = None + c.wfile = None + pc = protobuf._dump_http_client_conn(c) + lc = protobuf._load_http_client_conn(pc) + assert c.__dict__ == lc.__dict__ + + def test_roundtrip_client_cert(self, tdata): + c = tflow.tclient_conn() + c.rfile = None + c.wfile = None + del c.reply + with open(tdata.path("mitmproxy/net/data/clientcert/client.pem"), "rb") as f: + d = f.read() + c.clientcert = certs.Cert.from_pem(d) + pc = protobuf._dump_http_client_conn(c) + lc = protobuf._load_http_client_conn(pc) + assert c.__dict__ == lc.__dict__ + + def test_roundtrip_server(self): + s = tflow.tserver_conn() + del s.reply + s.wfile = None + s.rfile = None + ps = protobuf._dump_http_server_conn(s) + ls = protobuf._load_http_server_conn(ps) + assert s.__dict__ == ls.__dict__ + + def test_roundtrip_server_cert(self, tdata): + s = tflow.tserver_conn() + del s.reply + s.wfile = None + s.rfile = None + with open(tdata.path("mitmproxy/net/data/text_cert"), "rb") as f: + d = f.read() + s.cert = certs.Cert.from_pem(d) + ps = protobuf._dump_http_server_conn(s) + ls = protobuf._load_http_server_conn(ps) + assert s.__dict__ == ls.__dict__ + + def test_roundtrip_server_via(self): + s = tflow.tserver_conn() + s.via = tflow.tserver_conn() + del s.reply + s.wfile = None + s.rfile = None + ps = protobuf._dump_http_server_conn(s) + ls = protobuf._load_http_server_conn(ps) + assert s.__dict__ == ls.__dict__ + del s.via.reply + s.via.wfile = None + s.via.rfile = None + assert s.via.__dict__ == ls.via.__dict__ + + def test_roundtrip_http_request(self): + req = http.HTTPRequest.wrap(tutils.treq()) + preq = protobuf._dump_http_request(req) + lreq = protobuf._load_http_request(preq) + assert req.__dict__ == lreq.__dict__ + + def test_roundtrip_http_request_empty_content(self): + req = http.HTTPRequest.wrap(tutils.treq(content=b"")) + preq = protobuf._dump_http_request(req) + lreq = protobuf._load_http_request(preq) + assert req.__dict__ == lreq.__dict__ + + def test_roundtrip_http_response(self): + res = http.HTTPResponse.wrap(tutils.tresp()) + pres = protobuf._dump_http_response(res) + lres = protobuf._load_http_response(pres) + assert res.__dict__ == lres.__dict__ + + def test_roundtrip_http_response_empty_content(self): + res = http.HTTPResponse.wrap(tutils.tresp(content=b"")) + pres = protobuf._dump_http_response(res) + lres = protobuf._load_http_response(pres) + assert res.__dict__ == lres.__dict__ + + def test_roundtrip_http_error(self): + err = tflow.terr() + perr = protobuf._dump_http_error(err) + lerr = protobuf._load_http_error(perr) + assert err.__dict__ == lerr.__dict__ + + def test_roundtrip_http_flow_only_req(self): + f = tflow.tflow() + f.reply = None + pf = protobuf.dumps(f) + lf = protobuf.loads(pf, "http") + assert f.__dict__ == lf.__dict__ + + def test_roundtrip_http_flow_res(self): + f = tflow.tflow(resp=True) + f.reply = None + pf = protobuf.dumps(f) + lf = protobuf.loads(pf, "http") + assert f.__dict__ == lf.__dict__ + + def test_unsupported_dumps(self): + w = tflow.twebsocketflow() + with pytest.raises(exceptions.TypeError): + protobuf.dumps(w) + + def test_unsupported_loads(self): + b = b"blobs" + with pytest.raises(exceptions.TypeError): + protobuf.loads(b, 'not-http')