From e842aa3798ee71812cc68ce0f3b7360edbc023c3 Mon Sep 17 00:00:00 2001 From: madt1m Date: Sat, 23 Jun 2018 03:02:39 +0200 Subject: [PATCH] Added protobuf methods; implemented initial DB logic --- examples/addons/protodumper.py | 5 + mitmproxy/io/db.py | 32 ++ mitmproxy/io/proto/connection.proto | 38 -- mitmproxy/io/proto/http.proto | 68 ++- mitmproxy/io/proto/http_pb2.py | 724 ++++++++++++++++++++++++++++ mitmproxy/io/protobuf.py | 60 ++- setup.py | 1 + 7 files changed, 868 insertions(+), 60 deletions(-) create mode 100644 examples/addons/protodumper.py delete mode 100644 mitmproxy/io/proto/connection.proto create mode 100644 mitmproxy/io/proto/http_pb2.py diff --git a/examples/addons/protodumper.py b/examples/addons/protodumper.py new file mode 100644 index 000000000..6a1d676c4 --- /dev/null +++ b/examples/addons/protodumper.py @@ -0,0 +1,5 @@ +from mitmproxy.io import protobuf + + +def response(f): + protobuf.loads(protobuf.dumps(f)) diff --git a/mitmproxy/io/db.py b/mitmproxy/io/db.py index e69de29bb..f7e13e915 100644 --- a/mitmproxy/io/db.py +++ b/mitmproxy/io/db.py @@ -0,0 +1,32 @@ +import sqlite3 +from mitmproxy.io import protobuf + + +class DbHandler: + + """ + This class is wrapping up connection to SQLITE DB. + """ + + def __init__(self, db_path="tmp.sqlite"): + self.db_path = db_path + self._con = sqlite3.connect(self.db_path) + self._c = self._con.cursor() + self._create_db() + + def _create_db(self): + with self._con: + self._con.execute('CREATE TABLE IF NOT EXISTS FLOWS(' + 'id INTEGER PRIMARY KEY AUTOINCREMENT,' + 'pbuf_blob BLOB)') + + def store(self, flows): + blobs = [] + for flow in flows: + blobs.append(protobuf.dumps(flow)) + with self._con: + self._con.executemany('INSERT INTO FLOWS values (?)', blobs) + + def load(self): + self._c.execute('SELECT * FROM FLOWS') + return self._c.fetchall() diff --git a/mitmproxy/io/proto/connection.proto b/mitmproxy/io/proto/connection.proto deleted file mode 100644 index fdcb0d137..000000000 --- a/mitmproxy/io/proto/connection.proto +++ /dev/null @@ -1,38 +0,0 @@ -syntax='proto2'; - -message Address { - optional string host = 1; - optional int32 port = 2; -} - -message ClientConnection { - optional string id = 1; - optional Address address = 2; - optional bool tls_established = 3; - optional string clientcert = 4; - optional string mitmcert = 5; - optional float timestamp_start = 6; - optional float timestamp_tls_setup = 7; - optional float timestamp_end = 8; - optional string sni = 9; - optional string cipher_name = 10; - optional bytes alpn_proto_negotiated = 11; - optional string tls_version = 12; -} - -message ServerConnection { - optional string id = 1; - optional Address address = 2; - optional Address ip_address = 3; - optional Address source_address = 4; - optional bool tls_established = 5; - optional string cert = 6; - optional string sni = 7; - optional bytes alpn_proto_negotiated = 8; - optional string tls_version = 9; - optional float timestamp_start = 10; - optional float timestamp_tcp_setup = 11; - optional float timestamp_tls_setup = 12; - optional float timestamp_end = 13; -} - diff --git a/mitmproxy/io/proto/http.proto b/mitmproxy/io/proto/http.proto index f1b90f78b..3db5333cb 100644 --- a/mitmproxy/io/proto/http.proto +++ b/mitmproxy/io/proto/http.proto @@ -1,5 +1,4 @@ syntax='proto2'; -import "connection.proto"; message HTTPFlow { optional HTTPRequest request = 1; @@ -16,36 +15,73 @@ message HTTPFlow { message HTTPRequest { optional string first_line_format = 1; - optional bytes method = 2; - optional bytes scheme = 3; - optional bytes host = 4; + optional string method = 2; + optional string scheme = 3; + optional string host = 4; optional int32 port = 5; - optional bytes path = 6; - optional bytes http_version = 7; + optional string path = 6; + optional string http_version = 7; repeated HTTPHeader headers = 8; optional bytes content = 9; - optional int64 timestamp_start = 10; - optional int64 timestamp_end = 11; + optional float timestamp_start = 10; + optional float timestamp_end = 11; optional bool is_replay = 12; } message HTTPResponse { - optional bytes http_version = 1; + optional string http_version = 1; optional int32 status_code = 2; - optional bytes reason = 3; + optional string reason = 3; repeated HTTPHeader headers = 4; optional bytes content = 5; - optional int64 timestamp_start = 6; - optional int64 timestamp_end = 7; + optional float timestamp_start = 6; + optional float timestamp_end = 7; optional bool is_replay = 8; } message HTTPError { optional string msg = 1; - optional int64 timestamp = 2; + optional float timestamp = 2; } message HTTPHeader { - optional bytes name = 1; - optional bytes value = 2; -} \ No newline at end of file + optional string name = 1; + optional string value = 2; +} + + +message Address { + optional string host = 1; + optional int32 port = 2; +} + +message ClientConnection { + optional string id = 1; + optional Address address = 2; + optional bool tls_established = 3; + optional string clientcert = 4; + optional string mitmcert = 5; + optional float timestamp_start = 6; + optional float timestamp_tls_setup = 7; + optional float timestamp_end = 8; + optional string sni = 9; + optional string cipher_name = 10; + optional bytes alpn_proto_negotiated = 11; + optional string tls_version = 12; +} + +message ServerConnection { + optional string id = 1; + optional Address address = 2; + optional Address ip_address = 3; + optional Address source_address = 4; + optional bool tls_established = 5; + optional string cert = 6; + optional string sni = 7; + optional bytes alpn_proto_negotiated = 8; + optional string tls_version = 9; + optional float timestamp_start = 10; + optional float timestamp_tcp_setup = 11; + optional float timestamp_tls_setup = 12; + optional float timestamp_end = 13; +} diff --git a/mitmproxy/io/proto/http_pb2.py b/mitmproxy/io/proto/http_pb2.py new file mode 100644 index 000000000..b1bcfbb6a --- /dev/null +++ b/mitmproxy/io/proto/http_pb2.py @@ -0,0 +1,724 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: http.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='http.proto', + package='', + syntax='proto2', + serialized_pb=_b('\n\nhttp.proto\"\x85\x02\n\x08HTTPFlow\x12\x1d\n\x07request\x18\x01 \x01(\x0b\x32\x0c.HTTPRequest\x12\x1f\n\x08response\x18\x02 \x01(\x0b\x32\r.HTTPResponse\x12\x19\n\x05\x65rror\x18\x03 \x01(\x0b\x32\n.HTTPError\x12&\n\x0b\x63lient_conn\x18\x04 \x01(\x0b\x32\x11.ClientConnection\x12&\n\x0bserver_conn\x18\x05 \x01(\x0b\x32\x11.ServerConnection\x12\x13\n\x0bintercepted\x18\x06 \x01(\x08\x12\x0e\n\x06marked\x18\x07 \x01(\x08\x12\x0c\n\x04mode\x18\x08 \x01(\t\x12\n\n\x02id\x18\t \x01(\t\x12\x0f\n\x07version\x18\n \x01(\x05\"\xfa\x01\n\x0bHTTPRequest\x12\x19\n\x11\x66irst_line_format\x18\x01 \x01(\t\x12\x0e\n\x06method\x18\x02 \x01(\t\x12\x0e\n\x06scheme\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\x05\x12\x0c\n\x04path\x18\x06 \x01(\t\x12\x14\n\x0chttp_version\x18\x07 \x01(\t\x12\x1c\n\x07headers\x18\x08 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\t \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x15\n\rtimestamp_end\x18\x0b \x01(\x02\x12\x11\n\tis_replay\x18\x0c \x01(\x08\"\xbb\x01\n\x0cHTTPResponse\x12\x14\n\x0chttp_version\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x1c\n\x07headers\x18\x04 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x07 \x01(\x02\x12\x11\n\tis_replay\x18\x08 \x01(\x08\"+\n\tHTTPError\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x02\")\n\nHTTPHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"%\n\x07\x41\x64\x64ress\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\x9b\x02\n\x10\x43lientConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x03 \x01(\x08\x12\x12\n\nclientcert\x18\x04 \x01(\t\x12\x10\n\x08mitmcert\x18\x05 \x01(\t\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x07 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x08 \x01(\x02\x12\x0b\n\x03sni\x18\t \x01(\t\x12\x13\n\x0b\x63ipher_name\x18\n \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x0b \x01(\x0c\x12\x13\n\x0btls_version\x18\x0c \x01(\t\"\xcb\x02\n\x10ServerConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x1c\n\nip_address\x18\x03 \x01(\x0b\x32\x08.Address\x12 \n\x0esource_address\x18\x04 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x05 \x01(\x08\x12\x0c\n\x04\x63\x65rt\x18\x06 \x01(\t\x12\x0b\n\x03sni\x18\x07 \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x08 \x01(\x0c\x12\x13\n\x0btls_version\x18\t \x01(\t\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x1b\n\x13timestamp_tcp_setup\x18\x0b \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x0c \x01(\x02\x12\x15\n\rtimestamp_end\x18\r \x01(\x02') +) + + + + +_HTTPFLOW = _descriptor.Descriptor( + name='HTTPFlow', + full_name='HTTPFlow', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='request', full_name='HTTPFlow.request', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='response', full_name='HTTPFlow.response', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='error', full_name='HTTPFlow.error', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='client_conn', full_name='HTTPFlow.client_conn', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='server_conn', full_name='HTTPFlow.server_conn', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='intercepted', full_name='HTTPFlow.intercepted', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='marked', full_name='HTTPFlow.marked', index=6, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mode', full_name='HTTPFlow.mode', index=7, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='id', full_name='HTTPFlow.id', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='version', full_name='HTTPFlow.version', index=9, + number=10, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=15, + serialized_end=276, +) + + +_HTTPREQUEST = _descriptor.Descriptor( + name='HTTPRequest', + full_name='HTTPRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='first_line_format', full_name='HTTPRequest.first_line_format', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='method', full_name='HTTPRequest.method', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='scheme', full_name='HTTPRequest.scheme', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='host', full_name='HTTPRequest.host', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='port', full_name='HTTPRequest.port', index=4, + number=5, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='path', full_name='HTTPRequest.path', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='http_version', full_name='HTTPRequest.http_version', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='headers', full_name='HTTPRequest.headers', index=7, + number=8, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='content', full_name='HTTPRequest.content', index=8, + number=9, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='HTTPRequest.timestamp_start', index=9, + number=10, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='HTTPRequest.timestamp_end', index=10, + number=11, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='is_replay', full_name='HTTPRequest.is_replay', index=11, + number=12, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=279, + serialized_end=529, +) + + +_HTTPRESPONSE = _descriptor.Descriptor( + name='HTTPResponse', + full_name='HTTPResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='http_version', full_name='HTTPResponse.http_version', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='status_code', full_name='HTTPResponse.status_code', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='reason', full_name='HTTPResponse.reason', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='headers', full_name='HTTPResponse.headers', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='content', full_name='HTTPResponse.content', index=4, + number=5, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='HTTPResponse.timestamp_start', index=5, + number=6, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='HTTPResponse.timestamp_end', index=6, + number=7, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='is_replay', full_name='HTTPResponse.is_replay', index=7, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=532, + serialized_end=719, +) + + +_HTTPERROR = _descriptor.Descriptor( + name='HTTPError', + full_name='HTTPError', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='msg', full_name='HTTPError.msg', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp', full_name='HTTPError.timestamp', index=1, + number=2, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=721, + serialized_end=764, +) + + +_HTTPHEADER = _descriptor.Descriptor( + name='HTTPHeader', + full_name='HTTPHeader', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='HTTPHeader.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value', full_name='HTTPHeader.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=766, + serialized_end=807, +) + + +_ADDRESS = _descriptor.Descriptor( + name='Address', + full_name='Address', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='host', full_name='Address.host', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='port', full_name='Address.port', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=809, + serialized_end=846, +) + + +_CLIENTCONNECTION = _descriptor.Descriptor( + name='ClientConnection', + full_name='ClientConnection', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='ClientConnection.id', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='address', full_name='ClientConnection.address', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_established', full_name='ClientConnection.tls_established', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='clientcert', full_name='ClientConnection.clientcert', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='mitmcert', full_name='ClientConnection.mitmcert', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='ClientConnection.timestamp_start', index=5, + number=6, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_tls_setup', full_name='ClientConnection.timestamp_tls_setup', index=6, + number=7, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='ClientConnection.timestamp_end', index=7, + number=8, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='sni', full_name='ClientConnection.sni', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='cipher_name', full_name='ClientConnection.cipher_name', index=9, + number=10, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='alpn_proto_negotiated', full_name='ClientConnection.alpn_proto_negotiated', index=10, + number=11, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_version', full_name='ClientConnection.tls_version', index=11, + number=12, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=849, + serialized_end=1132, +) + + +_SERVERCONNECTION = _descriptor.Descriptor( + name='ServerConnection', + full_name='ServerConnection', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='ServerConnection.id', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='address', full_name='ServerConnection.address', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='ip_address', full_name='ServerConnection.ip_address', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='source_address', full_name='ServerConnection.source_address', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_established', full_name='ServerConnection.tls_established', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='cert', full_name='ServerConnection.cert', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='sni', full_name='ServerConnection.sni', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='alpn_proto_negotiated', full_name='ServerConnection.alpn_proto_negotiated', index=7, + number=8, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='tls_version', full_name='ServerConnection.tls_version', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_start', full_name='ServerConnection.timestamp_start', index=9, + number=10, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_tcp_setup', full_name='ServerConnection.timestamp_tcp_setup', index=10, + number=11, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_tls_setup', full_name='ServerConnection.timestamp_tls_setup', index=11, + number=12, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='timestamp_end', full_name='ServerConnection.timestamp_end', index=12, + number=13, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1135, + serialized_end=1466, +) + +_HTTPFLOW.fields_by_name['request'].message_type = _HTTPREQUEST +_HTTPFLOW.fields_by_name['response'].message_type = _HTTPRESPONSE +_HTTPFLOW.fields_by_name['error'].message_type = _HTTPERROR +_HTTPFLOW.fields_by_name['client_conn'].message_type = _CLIENTCONNECTION +_HTTPFLOW.fields_by_name['server_conn'].message_type = _SERVERCONNECTION +_HTTPREQUEST.fields_by_name['headers'].message_type = _HTTPHEADER +_HTTPRESPONSE.fields_by_name['headers'].message_type = _HTTPHEADER +_CLIENTCONNECTION.fields_by_name['address'].message_type = _ADDRESS +_SERVERCONNECTION.fields_by_name['address'].message_type = _ADDRESS +_SERVERCONNECTION.fields_by_name['ip_address'].message_type = _ADDRESS +_SERVERCONNECTION.fields_by_name['source_address'].message_type = _ADDRESS +DESCRIPTOR.message_types_by_name['HTTPFlow'] = _HTTPFLOW +DESCRIPTOR.message_types_by_name['HTTPRequest'] = _HTTPREQUEST +DESCRIPTOR.message_types_by_name['HTTPResponse'] = _HTTPRESPONSE +DESCRIPTOR.message_types_by_name['HTTPError'] = _HTTPERROR +DESCRIPTOR.message_types_by_name['HTTPHeader'] = _HTTPHEADER +DESCRIPTOR.message_types_by_name['Address'] = _ADDRESS +DESCRIPTOR.message_types_by_name['ClientConnection'] = _CLIENTCONNECTION +DESCRIPTOR.message_types_by_name['ServerConnection'] = _SERVERCONNECTION +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +HTTPFlow = _reflection.GeneratedProtocolMessageType('HTTPFlow', (_message.Message,), dict( + DESCRIPTOR = _HTTPFLOW, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPFlow) + )) +_sym_db.RegisterMessage(HTTPFlow) + +HTTPRequest = _reflection.GeneratedProtocolMessageType('HTTPRequest', (_message.Message,), dict( + DESCRIPTOR = _HTTPREQUEST, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPRequest) + )) +_sym_db.RegisterMessage(HTTPRequest) + +HTTPResponse = _reflection.GeneratedProtocolMessageType('HTTPResponse', (_message.Message,), dict( + DESCRIPTOR = _HTTPRESPONSE, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPResponse) + )) +_sym_db.RegisterMessage(HTTPResponse) + +HTTPError = _reflection.GeneratedProtocolMessageType('HTTPError', (_message.Message,), dict( + DESCRIPTOR = _HTTPERROR, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPError) + )) +_sym_db.RegisterMessage(HTTPError) + +HTTPHeader = _reflection.GeneratedProtocolMessageType('HTTPHeader', (_message.Message,), dict( + DESCRIPTOR = _HTTPHEADER, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:HTTPHeader) + )) +_sym_db.RegisterMessage(HTTPHeader) + +Address = _reflection.GeneratedProtocolMessageType('Address', (_message.Message,), dict( + DESCRIPTOR = _ADDRESS, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:Address) + )) +_sym_db.RegisterMessage(Address) + +ClientConnection = _reflection.GeneratedProtocolMessageType('ClientConnection', (_message.Message,), dict( + DESCRIPTOR = _CLIENTCONNECTION, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:ClientConnection) + )) +_sym_db.RegisterMessage(ClientConnection) + +ServerConnection = _reflection.GeneratedProtocolMessageType('ServerConnection', (_message.Message,), dict( + DESCRIPTOR = _SERVERCONNECTION, + __module__ = 'http_pb2' + # @@protoc_insertion_point(class_scope:ServerConnection) + )) +_sym_db.RegisterMessage(ServerConnection) + + +# @@protoc_insertion_point(module_scope) diff --git a/mitmproxy/io/protobuf.py b/mitmproxy/io/protobuf.py index 2dd0f1e7d..dc526d360 100644 --- a/mitmproxy/io/protobuf.py +++ b/mitmproxy/io/protobuf.py @@ -1,11 +1,59 @@ +from mitmproxy import flow +from mitmproxy import exceptions +from mitmproxy.io.proto import http_pb2 -"Just define dumps and loads. Files will be handled later. Or maybe...do it, and DONT use it." +def dumps(f: flow.Flow) -> bytes: + if f.type != "http": + raise exceptions.TypeError("Flow types different than HTTP not supported yet!") + else: + hf = http_pb2.HTTPFlow() + state = f.get_state() + for r in ['request', 'response']: + for field in state[r]: + if hasattr(getattr(hf, r), field) and state[r][field] is not None: + if field == 'headers': + for n, v in state[r][field]: + header = (getattr(hf, r)).headers.add() + header.name = n + header.value = v + else: + setattr(getattr(hf, r), field, state[r][field]) + if state['error']: + hf.error.msg = state['error']['msg'] + hf.error.timestamp = state['error']['timestamp'] + for c in ['client_conn', 'server_conn']: + for field in state[c]: + if hasattr(getattr(hf, c), field) and state[c][field] is not None: + if field in ['address', 'ip_address', 'source_address']: + getattr(getattr(hf, c), field).host = state[c][field][0] + getattr(getattr(hf, c), field).port = state[c][field][1] + else: + setattr(getattr(hf, c), field, state[c][field]) + for val in ['intercepted', 'marked', 'mode', 'id', 'version']: + if state[val] is not None: + setattr(hf, val, state[val]) + return hf.SerializeToString() -import typing -def dumps(value) -> bytes: - pass +def loads(blob: bytes) -> flow.Flow: + r = http_pb2.HTTPFlow() + state = dict() + r.ParseFromString(blob) + _dump_object(r, state) -def dump(value, file_handle: typing.BinaryIO) -> None: - pass + +def _dump_object(obj, d): + for descriptor in obj.DESCRIPTOR.fields: + value = getattr(obj, descriptor.name) + if descriptor.type == descriptor.TYPE_MESSAGE: + d[descriptor.name] = {} + if descriptor.label == descriptor.LABEL_REPEATED: + [_dump_object(v, d[descriptor.name]) for v in value] + else: + _dump_object(value, d[descriptor.name]) + elif descriptor.type == descriptor.TYPE_ENUM: + enum_name = descriptor.enum_type.values[value].name + d[descriptor.name] = enum_name + else: + d[descriptor.name] = value diff --git a/setup.py b/setup.py index 6ba9391e2..acd3f2952 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ setup( "pyOpenSSL>=17.5,<18.1", "pyparsing>=2.1.3, <2.3", "pyperclip>=1.6.0, <1.7", + "protobuf>=3.6.0, <3.7", "ruamel.yaml>=0.13.2, <0.16", "sortedcontainers>=1.5.4,<2.1", "tornado>=4.3,<5.1",