Added protobuf methods; implemented initial DB logic

This commit is contained in:
madt1m 2018-06-23 03:02:39 +02:00
parent ba144b15bc
commit e842aa3798
7 changed files with 868 additions and 60 deletions

View File

@ -0,0 +1,5 @@
from mitmproxy.io import protobuf
def response(f):
protobuf.loads(protobuf.dumps(f))

View File

@ -0,0 +1,32 @@
import sqlite3
from mitmproxy.io import protobuf
class DbHandler:
"""
This class is wrapping up connection to SQLITE DB.
"""
def __init__(self, db_path="tmp.sqlite"):
self.db_path = db_path
self._con = sqlite3.connect(self.db_path)
self._c = self._con.cursor()
self._create_db()
def _create_db(self):
with self._con:
self._con.execute('CREATE TABLE IF NOT EXISTS FLOWS('
'id INTEGER PRIMARY KEY AUTOINCREMENT,'
'pbuf_blob BLOB)')
def store(self, flows):
blobs = []
for flow in flows:
blobs.append(protobuf.dumps(flow))
with self._con:
self._con.executemany('INSERT INTO FLOWS values (?)', blobs)
def load(self):
self._c.execute('SELECT * FROM FLOWS')
return self._c.fetchall()

View File

@ -1,38 +0,0 @@
syntax='proto2';
message Address {
optional string host = 1;
optional int32 port = 2;
}
message ClientConnection {
optional string id = 1;
optional Address address = 2;
optional bool tls_established = 3;
optional string clientcert = 4;
optional string mitmcert = 5;
optional float timestamp_start = 6;
optional float timestamp_tls_setup = 7;
optional float timestamp_end = 8;
optional string sni = 9;
optional string cipher_name = 10;
optional bytes alpn_proto_negotiated = 11;
optional string tls_version = 12;
}
message ServerConnection {
optional string id = 1;
optional Address address = 2;
optional Address ip_address = 3;
optional Address source_address = 4;
optional bool tls_established = 5;
optional string cert = 6;
optional string sni = 7;
optional bytes alpn_proto_negotiated = 8;
optional string tls_version = 9;
optional float timestamp_start = 10;
optional float timestamp_tcp_setup = 11;
optional float timestamp_tls_setup = 12;
optional float timestamp_end = 13;
}

View File

@ -1,5 +1,4 @@
syntax='proto2';
import "connection.proto";
message HTTPFlow {
optional HTTPRequest request = 1;
@ -16,36 +15,73 @@ message HTTPFlow {
message HTTPRequest {
optional string first_line_format = 1;
optional bytes method = 2;
optional bytes scheme = 3;
optional bytes host = 4;
optional string method = 2;
optional string scheme = 3;
optional string host = 4;
optional int32 port = 5;
optional bytes path = 6;
optional bytes http_version = 7;
optional string path = 6;
optional string http_version = 7;
repeated HTTPHeader headers = 8;
optional bytes content = 9;
optional int64 timestamp_start = 10;
optional int64 timestamp_end = 11;
optional float timestamp_start = 10;
optional float timestamp_end = 11;
optional bool is_replay = 12;
}
message HTTPResponse {
optional bytes http_version = 1;
optional string http_version = 1;
optional int32 status_code = 2;
optional bytes reason = 3;
optional string reason = 3;
repeated HTTPHeader headers = 4;
optional bytes content = 5;
optional int64 timestamp_start = 6;
optional int64 timestamp_end = 7;
optional float timestamp_start = 6;
optional float timestamp_end = 7;
optional bool is_replay = 8;
}
message HTTPError {
optional string msg = 1;
optional int64 timestamp = 2;
optional float timestamp = 2;
}
message HTTPHeader {
optional bytes name = 1;
optional bytes value = 2;
optional string name = 1;
optional string value = 2;
}
message Address {
optional string host = 1;
optional int32 port = 2;
}
message ClientConnection {
optional string id = 1;
optional Address address = 2;
optional bool tls_established = 3;
optional string clientcert = 4;
optional string mitmcert = 5;
optional float timestamp_start = 6;
optional float timestamp_tls_setup = 7;
optional float timestamp_end = 8;
optional string sni = 9;
optional string cipher_name = 10;
optional bytes alpn_proto_negotiated = 11;
optional string tls_version = 12;
}
message ServerConnection {
optional string id = 1;
optional Address address = 2;
optional Address ip_address = 3;
optional Address source_address = 4;
optional bool tls_established = 5;
optional string cert = 6;
optional string sni = 7;
optional bytes alpn_proto_negotiated = 8;
optional string tls_version = 9;
optional float timestamp_start = 10;
optional float timestamp_tcp_setup = 11;
optional float timestamp_tls_setup = 12;
optional float timestamp_end = 13;
}

View File

@ -0,0 +1,724 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: http.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='http.proto',
package='',
syntax='proto2',
serialized_pb=_b('\n\nhttp.proto\"\x85\x02\n\x08HTTPFlow\x12\x1d\n\x07request\x18\x01 \x01(\x0b\x32\x0c.HTTPRequest\x12\x1f\n\x08response\x18\x02 \x01(\x0b\x32\r.HTTPResponse\x12\x19\n\x05\x65rror\x18\x03 \x01(\x0b\x32\n.HTTPError\x12&\n\x0b\x63lient_conn\x18\x04 \x01(\x0b\x32\x11.ClientConnection\x12&\n\x0bserver_conn\x18\x05 \x01(\x0b\x32\x11.ServerConnection\x12\x13\n\x0bintercepted\x18\x06 \x01(\x08\x12\x0e\n\x06marked\x18\x07 \x01(\x08\x12\x0c\n\x04mode\x18\x08 \x01(\t\x12\n\n\x02id\x18\t \x01(\t\x12\x0f\n\x07version\x18\n \x01(\x05\"\xfa\x01\n\x0bHTTPRequest\x12\x19\n\x11\x66irst_line_format\x18\x01 \x01(\t\x12\x0e\n\x06method\x18\x02 \x01(\t\x12\x0e\n\x06scheme\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\x05\x12\x0c\n\x04path\x18\x06 \x01(\t\x12\x14\n\x0chttp_version\x18\x07 \x01(\t\x12\x1c\n\x07headers\x18\x08 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\t \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x15\n\rtimestamp_end\x18\x0b \x01(\x02\x12\x11\n\tis_replay\x18\x0c \x01(\x08\"\xbb\x01\n\x0cHTTPResponse\x12\x14\n\x0chttp_version\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x1c\n\x07headers\x18\x04 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x07 \x01(\x02\x12\x11\n\tis_replay\x18\x08 \x01(\x08\"+\n\tHTTPError\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x02\")\n\nHTTPHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"%\n\x07\x41\x64\x64ress\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\x9b\x02\n\x10\x43lientConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x03 \x01(\x08\x12\x12\n\nclientcert\x18\x04 \x01(\t\x12\x10\n\x08mitmcert\x18\x05 \x01(\t\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x07 \x01(\x02\x12\x15\n\rtimestamp_end\x18\x08 \x01(\x02\x12\x0b\n\x03sni\x18\t \x01(\t\x12\x13\n\x0b\x63ipher_name\x18\n \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x0b \x01(\x0c\x12\x13\n\x0btls_version\x18\x0c \x01(\t\"\xcb\x02\n\x10ServerConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x1c\n\nip_address\x18\x03 \x01(\x0b\x32\x08.Address\x12 \n\x0esource_address\x18\x04 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x05 \x01(\x08\x12\x0c\n\x04\x63\x65rt\x18\x06 \x01(\t\x12\x0b\n\x03sni\x18\x07 \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x08 \x01(\x0c\x12\x13\n\x0btls_version\x18\t \x01(\t\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x02\x12\x1b\n\x13timestamp_tcp_setup\x18\x0b \x01(\x02\x12\x1b\n\x13timestamp_tls_setup\x18\x0c \x01(\x02\x12\x15\n\rtimestamp_end\x18\r \x01(\x02')
)
_HTTPFLOW = _descriptor.Descriptor(
name='HTTPFlow',
full_name='HTTPFlow',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='request', full_name='HTTPFlow.request', index=0,
number=1, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='response', full_name='HTTPFlow.response', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='error', full_name='HTTPFlow.error', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='client_conn', full_name='HTTPFlow.client_conn', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='server_conn', full_name='HTTPFlow.server_conn', index=4,
number=5, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='intercepted', full_name='HTTPFlow.intercepted', index=5,
number=6, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='marked', full_name='HTTPFlow.marked', index=6,
number=7, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='mode', full_name='HTTPFlow.mode', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='id', full_name='HTTPFlow.id', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='version', full_name='HTTPFlow.version', index=9,
number=10, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=15,
serialized_end=276,
)
_HTTPREQUEST = _descriptor.Descriptor(
name='HTTPRequest',
full_name='HTTPRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='first_line_format', full_name='HTTPRequest.first_line_format', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='method', full_name='HTTPRequest.method', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='scheme', full_name='HTTPRequest.scheme', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='host', full_name='HTTPRequest.host', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='port', full_name='HTTPRequest.port', index=4,
number=5, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='path', full_name='HTTPRequest.path', index=5,
number=6, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='http_version', full_name='HTTPRequest.http_version', index=6,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='headers', full_name='HTTPRequest.headers', index=7,
number=8, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='content', full_name='HTTPRequest.content', index=8,
number=9, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_start', full_name='HTTPRequest.timestamp_start', index=9,
number=10, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_end', full_name='HTTPRequest.timestamp_end', index=10,
number=11, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='is_replay', full_name='HTTPRequest.is_replay', index=11,
number=12, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=279,
serialized_end=529,
)
_HTTPRESPONSE = _descriptor.Descriptor(
name='HTTPResponse',
full_name='HTTPResponse',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='http_version', full_name='HTTPResponse.http_version', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='status_code', full_name='HTTPResponse.status_code', index=1,
number=2, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='reason', full_name='HTTPResponse.reason', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='headers', full_name='HTTPResponse.headers', index=3,
number=4, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='content', full_name='HTTPResponse.content', index=4,
number=5, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_start', full_name='HTTPResponse.timestamp_start', index=5,
number=6, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_end', full_name='HTTPResponse.timestamp_end', index=6,
number=7, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='is_replay', full_name='HTTPResponse.is_replay', index=7,
number=8, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=532,
serialized_end=719,
)
_HTTPERROR = _descriptor.Descriptor(
name='HTTPError',
full_name='HTTPError',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='msg', full_name='HTTPError.msg', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp', full_name='HTTPError.timestamp', index=1,
number=2, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=721,
serialized_end=764,
)
_HTTPHEADER = _descriptor.Descriptor(
name='HTTPHeader',
full_name='HTTPHeader',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='name', full_name='HTTPHeader.name', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='value', full_name='HTTPHeader.value', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=766,
serialized_end=807,
)
_ADDRESS = _descriptor.Descriptor(
name='Address',
full_name='Address',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='host', full_name='Address.host', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='port', full_name='Address.port', index=1,
number=2, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=809,
serialized_end=846,
)
_CLIENTCONNECTION = _descriptor.Descriptor(
name='ClientConnection',
full_name='ClientConnection',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='ClientConnection.id', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='address', full_name='ClientConnection.address', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tls_established', full_name='ClientConnection.tls_established', index=2,
number=3, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='clientcert', full_name='ClientConnection.clientcert', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='mitmcert', full_name='ClientConnection.mitmcert', index=4,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_start', full_name='ClientConnection.timestamp_start', index=5,
number=6, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_tls_setup', full_name='ClientConnection.timestamp_tls_setup', index=6,
number=7, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_end', full_name='ClientConnection.timestamp_end', index=7,
number=8, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='sni', full_name='ClientConnection.sni', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='cipher_name', full_name='ClientConnection.cipher_name', index=9,
number=10, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='alpn_proto_negotiated', full_name='ClientConnection.alpn_proto_negotiated', index=10,
number=11, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tls_version', full_name='ClientConnection.tls_version', index=11,
number=12, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=849,
serialized_end=1132,
)
_SERVERCONNECTION = _descriptor.Descriptor(
name='ServerConnection',
full_name='ServerConnection',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='ServerConnection.id', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='address', full_name='ServerConnection.address', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='ip_address', full_name='ServerConnection.ip_address', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='source_address', full_name='ServerConnection.source_address', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tls_established', full_name='ServerConnection.tls_established', index=4,
number=5, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='cert', full_name='ServerConnection.cert', index=5,
number=6, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='sni', full_name='ServerConnection.sni', index=6,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='alpn_proto_negotiated', full_name='ServerConnection.alpn_proto_negotiated', index=7,
number=8, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tls_version', full_name='ServerConnection.tls_version', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_start', full_name='ServerConnection.timestamp_start', index=9,
number=10, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_tcp_setup', full_name='ServerConnection.timestamp_tcp_setup', index=10,
number=11, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_tls_setup', full_name='ServerConnection.timestamp_tls_setup', index=11,
number=12, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_end', full_name='ServerConnection.timestamp_end', index=12,
number=13, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=1135,
serialized_end=1466,
)
_HTTPFLOW.fields_by_name['request'].message_type = _HTTPREQUEST
_HTTPFLOW.fields_by_name['response'].message_type = _HTTPRESPONSE
_HTTPFLOW.fields_by_name['error'].message_type = _HTTPERROR
_HTTPFLOW.fields_by_name['client_conn'].message_type = _CLIENTCONNECTION
_HTTPFLOW.fields_by_name['server_conn'].message_type = _SERVERCONNECTION
_HTTPREQUEST.fields_by_name['headers'].message_type = _HTTPHEADER
_HTTPRESPONSE.fields_by_name['headers'].message_type = _HTTPHEADER
_CLIENTCONNECTION.fields_by_name['address'].message_type = _ADDRESS
_SERVERCONNECTION.fields_by_name['address'].message_type = _ADDRESS
_SERVERCONNECTION.fields_by_name['ip_address'].message_type = _ADDRESS
_SERVERCONNECTION.fields_by_name['source_address'].message_type = _ADDRESS
DESCRIPTOR.message_types_by_name['HTTPFlow'] = _HTTPFLOW
DESCRIPTOR.message_types_by_name['HTTPRequest'] = _HTTPREQUEST
DESCRIPTOR.message_types_by_name['HTTPResponse'] = _HTTPRESPONSE
DESCRIPTOR.message_types_by_name['HTTPError'] = _HTTPERROR
DESCRIPTOR.message_types_by_name['HTTPHeader'] = _HTTPHEADER
DESCRIPTOR.message_types_by_name['Address'] = _ADDRESS
DESCRIPTOR.message_types_by_name['ClientConnection'] = _CLIENTCONNECTION
DESCRIPTOR.message_types_by_name['ServerConnection'] = _SERVERCONNECTION
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
HTTPFlow = _reflection.GeneratedProtocolMessageType('HTTPFlow', (_message.Message,), dict(
DESCRIPTOR = _HTTPFLOW,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:HTTPFlow)
))
_sym_db.RegisterMessage(HTTPFlow)
HTTPRequest = _reflection.GeneratedProtocolMessageType('HTTPRequest', (_message.Message,), dict(
DESCRIPTOR = _HTTPREQUEST,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:HTTPRequest)
))
_sym_db.RegisterMessage(HTTPRequest)
HTTPResponse = _reflection.GeneratedProtocolMessageType('HTTPResponse', (_message.Message,), dict(
DESCRIPTOR = _HTTPRESPONSE,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:HTTPResponse)
))
_sym_db.RegisterMessage(HTTPResponse)
HTTPError = _reflection.GeneratedProtocolMessageType('HTTPError', (_message.Message,), dict(
DESCRIPTOR = _HTTPERROR,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:HTTPError)
))
_sym_db.RegisterMessage(HTTPError)
HTTPHeader = _reflection.GeneratedProtocolMessageType('HTTPHeader', (_message.Message,), dict(
DESCRIPTOR = _HTTPHEADER,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:HTTPHeader)
))
_sym_db.RegisterMessage(HTTPHeader)
Address = _reflection.GeneratedProtocolMessageType('Address', (_message.Message,), dict(
DESCRIPTOR = _ADDRESS,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:Address)
))
_sym_db.RegisterMessage(Address)
ClientConnection = _reflection.GeneratedProtocolMessageType('ClientConnection', (_message.Message,), dict(
DESCRIPTOR = _CLIENTCONNECTION,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:ClientConnection)
))
_sym_db.RegisterMessage(ClientConnection)
ServerConnection = _reflection.GeneratedProtocolMessageType('ServerConnection', (_message.Message,), dict(
DESCRIPTOR = _SERVERCONNECTION,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:ServerConnection)
))
_sym_db.RegisterMessage(ServerConnection)
# @@protoc_insertion_point(module_scope)

View File

@ -1,11 +1,59 @@
from mitmproxy import flow
from mitmproxy import exceptions
from mitmproxy.io.proto import http_pb2
"Just define dumps and loads. Files will be handled later. Or maybe...do it, and DONT use it."
def dumps(f: flow.Flow) -> bytes:
if f.type != "http":
raise exceptions.TypeError("Flow types different than HTTP not supported yet!")
else:
hf = http_pb2.HTTPFlow()
state = f.get_state()
for r in ['request', 'response']:
for field in state[r]:
if hasattr(getattr(hf, r), field) and state[r][field] is not None:
if field == 'headers':
for n, v in state[r][field]:
header = (getattr(hf, r)).headers.add()
header.name = n
header.value = v
else:
setattr(getattr(hf, r), field, state[r][field])
if state['error']:
hf.error.msg = state['error']['msg']
hf.error.timestamp = state['error']['timestamp']
for c in ['client_conn', 'server_conn']:
for field in state[c]:
if hasattr(getattr(hf, c), field) and state[c][field] is not None:
if field in ['address', 'ip_address', 'source_address']:
getattr(getattr(hf, c), field).host = state[c][field][0]
getattr(getattr(hf, c), field).port = state[c][field][1]
else:
setattr(getattr(hf, c), field, state[c][field])
for val in ['intercepted', 'marked', 'mode', 'id', 'version']:
if state[val] is not None:
setattr(hf, val, state[val])
return hf.SerializeToString()
import typing
def dumps(value) -> bytes:
pass
def loads(blob: bytes) -> flow.Flow:
r = http_pb2.HTTPFlow()
state = dict()
r.ParseFromString(blob)
_dump_object(r, state)
def dump(value, file_handle: typing.BinaryIO) -> None:
pass
def _dump_object(obj, d):
for descriptor in obj.DESCRIPTOR.fields:
value = getattr(obj, descriptor.name)
if descriptor.type == descriptor.TYPE_MESSAGE:
d[descriptor.name] = {}
if descriptor.label == descriptor.LABEL_REPEATED:
[_dump_object(v, d[descriptor.name]) for v in value]
else:
_dump_object(value, d[descriptor.name])
elif descriptor.type == descriptor.TYPE_ENUM:
enum_name = descriptor.enum_type.values[value].name
d[descriptor.name] = enum_name
else:
d[descriptor.name] = value

View File

@ -74,6 +74,7 @@ setup(
"pyOpenSSL>=17.5,<18.1",
"pyparsing>=2.1.3, <2.3",
"pyperclip>=1.6.0, <1.7",
"protobuf>=3.6.0, <3.7",
"ruamel.yaml>=0.13.2, <0.16",
"sortedcontainers>=1.5.4,<2.1",
"tornado>=4.3,<5.1",