Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2025-02-02 00:05:27 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3245 from madt1m/protobuf-serialization-clean

Browse Source 
Shifting to Protobuf Serialization - Cleaned
This commit is contained in:
Aldo Cortesi 2018-07-21 16:13:25 +12:00 committed by GitHub
commit ec092fdc12
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 1263 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -5,6 +5,7 @@ MANIFEST
*.py[cdo] *.py[cdo]
*.swp *.swp
*.swo *.swo
*.sqlite
*.egg-info/ *.egg-info/
.coverage* .coverage*
.idea .idea

3
mitmproxy/io/__init__.py
View File

@ -1,7 +1,8 @@
from .io import FlowWriter, FlowReader, FilteredFlowWriter, read_flows_from_paths from .io import FlowWriter, FlowReader, FilteredFlowWriter, read_flows_from_paths
from .db import DBHandler
__all__ = [ __all__ = [
"FlowWriter", "FlowReader", "FilteredFlowWriter", "read_flows_from_paths" "FlowWriter", "FlowReader", "FilteredFlowWriter", "read_flows_from_paths", "DBHandler"
] ]

40
mitmproxy/io/db.py Normal file
View File

@ -0,0 +1,40 @@
import sqlite3
import os
from mitmproxy.io import protobuf
class DBHandler:
"""
This class is wrapping up connection to SQLITE DB.
"""
def __init__(self, db_path, mode='load'):
if mode == 'write':
if os.path.isfile(db_path):
os.remove(db_path)
self.db_path = db_path
self._con = sqlite3.connect(self.db_path)
self._c = self._con.cursor()
self._create_db()
def _create_db(self):
with self._con:
self._con.execute('CREATE TABLE IF NOT EXISTS FLOWS('
'id INTEGER PRIMARY KEY,'
'pbuf_blob BLOB)')
def store(self, flows):
blobs = []
for flow in flows:
blobs.append((protobuf.dumps(flow),))
with self._con:
self._con.executemany('INSERT INTO FLOWS (pbuf_blob) values (?)', blobs)
def load(self):
flows = []
self._c.execute('SELECT pbuf_blob FROM FLOWS')
for row in self._c.fetchall():
flows.append((protobuf.loads(row[0])))
return flows

93
mitmproxy/io/proto/http.proto Normal file
View File

@ -0,0 +1,93 @@
syntax='proto2';
message HTTPFlow {
optional HTTPRequest request = 1;
optional HTTPResponse response = 2;
optional HTTPError error = 3;
optional ClientConnection client_conn = 4;
optional ServerConnection server_conn = 5;
optional bool intercepted = 6;
optional bool marked = 7;
optional string mode = 8;
optional string id = 9;
}
message HTTPRequest {
optional string first_line_format = 1;
optional string method = 2;
optional string scheme = 3;
optional string host = 4;
optional int32 port = 5;
optional string path = 6;
optional string http_version = 7;
repeated HTTPHeader headers = 8;
optional bytes content = 9;
optional double timestamp_start = 10;
optional double timestamp_end = 11;
optional bool is_replay = 12;
}
message HTTPResponse {
optional string http_version = 1;
optional int32 status_code = 2;
optional string reason = 3;
repeated HTTPHeader headers = 4;
optional bytes content = 5;
optional double timestamp_start = 6;
optional double timestamp_end = 7;
optional bool is_replay = 8;
}
message HTTPError {
optional string msg = 1;
optional double timestamp = 2;
}
message HTTPHeader {
optional string name = 1;
optional string value = 2;
}
message Address {
optional string host = 1;
optional int32 port = 2;
}
message ClientConnection {
optional string id = 1;
optional Address address = 2;
optional bool tls_established = 3;
optional string clientcert = 4;
optional string mitmcert = 5;
optional double timestamp_start = 6;
optional double timestamp_tls_setup = 7;
optional double timestamp_end = 8;
optional string sni = 9;
optional string cipher_name = 10;
optional bytes alpn_proto_negotiated = 11;
optional string tls_version = 12;
repeated TLSExtension tls_extensions = 13;
}
message ServerConnection {
optional string id = 1;
optional Address address = 2;
optional Address ip_address = 3;
optional Address source_address = 4;
optional bool tls_established = 5;
optional string cert = 6;
optional string sni = 7;
optional bytes alpn_proto_negotiated = 8;
optional string tls_version = 9;
optional double timestamp_start = 10;
optional double timestamp_tcp_setup = 11;
optional double timestamp_tls_setup = 12;
optional double timestamp_end = 13;
optional ServerConnection via = 14;
}
message TLSExtension {
optional int64 int = 1;
optional bytes bytes = 2;
}

779
mitmproxy/io/proto/http_pb2.py Normal file
View File

@ -0,0 +1,779 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: http.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='http.proto',
package='',
syntax='proto2',
serialized_pb=_b('\n\nhttp.proto\"\xf4\x01\n\x08HTTPFlow\x12\x1d\n\x07request\x18\x01 \x01(\x0b\x32\x0c.HTTPRequest\x12\x1f\n\x08response\x18\x02 \x01(\x0b\x32\r.HTTPResponse\x12\x19\n\x05\x65rror\x18\x03 \x01(\x0b\x32\n.HTTPError\x12&\n\x0b\x63lient_conn\x18\x04 \x01(\x0b\x32\x11.ClientConnection\x12&\n\x0bserver_conn\x18\x05 \x01(\x0b\x32\x11.ServerConnection\x12\x13\n\x0bintercepted\x18\x06 \x01(\x08\x12\x0e\n\x06marked\x18\x07 \x01(\x08\x12\x0c\n\x04mode\x18\x08 \x01(\t\x12\n\n\x02id\x18\t \x01(\t\"\xfa\x01\n\x0bHTTPRequest\x12\x19\n\x11\x66irst_line_format\x18\x01 \x01(\t\x12\x0e\n\x06method\x18\x02 \x01(\t\x12\x0e\n\x06scheme\x18\x03 \x01(\t\x12\x0c\n\x04host\x18\x04 \x01(\t\x12\x0c\n\x04port\x18\x05 \x01(\x05\x12\x0c\n\x04path\x18\x06 \x01(\t\x12\x14\n\x0chttp_version\x18\x07 \x01(\t\x12\x1c\n\x07headers\x18\x08 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\t \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x01\x12\x15\n\rtimestamp_end\x18\x0b \x01(\x01\x12\x11\n\tis_replay\x18\x0c \x01(\x08\"\xbb\x01\n\x0cHTTPResponse\x12\x14\n\x0chttp_version\x18\x01 \x01(\t\x12\x13\n\x0bstatus_code\x18\x02 \x01(\x05\x12\x0e\n\x06reason\x18\x03 \x01(\t\x12\x1c\n\x07headers\x18\x04 \x03(\x0b\x32\x0b.HTTPHeader\x12\x0f\n\x07\x63ontent\x18\x05 \x01(\x0c\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x01\x12\x15\n\rtimestamp_end\x18\x07 \x01(\x01\x12\x11\n\tis_replay\x18\x08 \x01(\x08\"+\n\tHTTPError\x12\x0b\n\x03msg\x18\x01 \x01(\t\x12\x11\n\ttimestamp\x18\x02 \x01(\x01\")\n\nHTTPHeader\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"%\n\x07\x41\x64\x64ress\x12\x0c\n\x04host\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xc2\x02\n\x10\x43lientConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x03 \x01(\x08\x12\x12\n\nclientcert\x18\x04 \x01(\t\x12\x10\n\x08mitmcert\x18\x05 \x01(\t\x12\x17\n\x0ftimestamp_start\x18\x06 \x01(\x01\x12\x1b\n\x13timestamp_tls_setup\x18\x07 \x01(\x01\x12\x15\n\rtimestamp_end\x18\x08 \x01(\x01\x12\x0b\n\x03sni\x18\t \x01(\t\x12\x13\n\x0b\x63ipher_name\x18\n \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x0b \x01(\x0c\x12\x13\n\x0btls_version\x18\x0c \x01(\t\x12%\n\x0etls_extensions\x18\r \x03(\x0b\x32\r.TLSExtension\"\xeb\x02\n\x10ServerConnection\x12\n\n\x02id\x18\x01 \x01(\t\x12\x19\n\x07\x61\x64\x64ress\x18\x02 \x01(\x0b\x32\x08.Address\x12\x1c\n\nip_address\x18\x03 \x01(\x0b\x32\x08.Address\x12 \n\x0esource_address\x18\x04 \x01(\x0b\x32\x08.Address\x12\x17\n\x0ftls_established\x18\x05 \x01(\x08\x12\x0c\n\x04\x63\x65rt\x18\x06 \x01(\t\x12\x0b\n\x03sni\x18\x07 \x01(\t\x12\x1d\n\x15\x61lpn_proto_negotiated\x18\x08 \x01(\x0c\x12\x13\n\x0btls_version\x18\t \x01(\t\x12\x17\n\x0ftimestamp_start\x18\n \x01(\x01\x12\x1b\n\x13timestamp_tcp_setup\x18\x0b \x01(\x01\x12\x1b\n\x13timestamp_tls_setup\x18\x0c \x01(\x01\x12\x15\n\rtimestamp_end\x18\r \x01(\x01\x12\x1e\n\x03via\x18\x0e \x01(\x0b\x32\x11.ServerConnection\"*\n\x0cTLSExtension\x12\x0b\n\x03int\x18\x01 \x01(\x03\x12\r\n\x05\x62ytes\x18\x02 \x01(\x0c')
)
_HTTPFLOW = _descriptor.Descriptor(
name='HTTPFlow',
full_name='HTTPFlow',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='request', full_name='HTTPFlow.request', index=0,
number=1, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='response', full_name='HTTPFlow.response', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='error', full_name='HTTPFlow.error', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='client_conn', full_name='HTTPFlow.client_conn', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='server_conn', full_name='HTTPFlow.server_conn', index=4,
number=5, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='intercepted', full_name='HTTPFlow.intercepted', index=5,
number=6, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='marked', full_name='HTTPFlow.marked', index=6,
number=7, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='mode', full_name='HTTPFlow.mode', index=7,
number=8, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='id', full_name='HTTPFlow.id', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=15,
serialized_end=259,
)
_HTTPREQUEST = _descriptor.Descriptor(
name='HTTPRequest',
full_name='HTTPRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='first_line_format', full_name='HTTPRequest.first_line_format', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='method', full_name='HTTPRequest.method', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='scheme', full_name='HTTPRequest.scheme', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='host', full_name='HTTPRequest.host', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='port', full_name='HTTPRequest.port', index=4,
number=5, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='path', full_name='HTTPRequest.path', index=5,
number=6, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='http_version', full_name='HTTPRequest.http_version', index=6,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='headers', full_name='HTTPRequest.headers', index=7,
number=8, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='content', full_name='HTTPRequest.content', index=8,
number=9, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_start', full_name='HTTPRequest.timestamp_start', index=9,
number=10, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_end', full_name='HTTPRequest.timestamp_end', index=10,
number=11, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='is_replay', full_name='HTTPRequest.is_replay', index=11,
number=12, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=262,
serialized_end=512,
)
_HTTPRESPONSE = _descriptor.Descriptor(
name='HTTPResponse',
full_name='HTTPResponse',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='http_version', full_name='HTTPResponse.http_version', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='status_code', full_name='HTTPResponse.status_code', index=1,
number=2, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='reason', full_name='HTTPResponse.reason', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='headers', full_name='HTTPResponse.headers', index=3,
number=4, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='content', full_name='HTTPResponse.content', index=4,
number=5, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_start', full_name='HTTPResponse.timestamp_start', index=5,
number=6, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_end', full_name='HTTPResponse.timestamp_end', index=6,
number=7, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='is_replay', full_name='HTTPResponse.is_replay', index=7,
number=8, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=515,
serialized_end=702,
)
_HTTPERROR = _descriptor.Descriptor(
name='HTTPError',
full_name='HTTPError',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='msg', full_name='HTTPError.msg', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp', full_name='HTTPError.timestamp', index=1,
number=2, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=704,
serialized_end=747,
)
_HTTPHEADER = _descriptor.Descriptor(
name='HTTPHeader',
full_name='HTTPHeader',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='name', full_name='HTTPHeader.name', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='value', full_name='HTTPHeader.value', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=749,
serialized_end=790,
)
_ADDRESS = _descriptor.Descriptor(
name='Address',
full_name='Address',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='host', full_name='Address.host', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='port', full_name='Address.port', index=1,
number=2, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=792,
serialized_end=829,
)
_CLIENTCONNECTION = _descriptor.Descriptor(
name='ClientConnection',
full_name='ClientConnection',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='ClientConnection.id', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='address', full_name='ClientConnection.address', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tls_established', full_name='ClientConnection.tls_established', index=2,
number=3, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='clientcert', full_name='ClientConnection.clientcert', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='mitmcert', full_name='ClientConnection.mitmcert', index=4,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_start', full_name='ClientConnection.timestamp_start', index=5,
number=6, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_tls_setup', full_name='ClientConnection.timestamp_tls_setup', index=6,
number=7, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_end', full_name='ClientConnection.timestamp_end', index=7,
number=8, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='sni', full_name='ClientConnection.sni', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='cipher_name', full_name='ClientConnection.cipher_name', index=9,
number=10, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='alpn_proto_negotiated', full_name='ClientConnection.alpn_proto_negotiated', index=10,
number=11, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tls_version', full_name='ClientConnection.tls_version', index=11,
number=12, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tls_extensions', full_name='ClientConnection.tls_extensions', index=12,
number=13, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=832,
serialized_end=1154,
)
_SERVERCONNECTION = _descriptor.Descriptor(
name='ServerConnection',
full_name='ServerConnection',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='ServerConnection.id', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='address', full_name='ServerConnection.address', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='ip_address', full_name='ServerConnection.ip_address', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='source_address', full_name='ServerConnection.source_address', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tls_established', full_name='ServerConnection.tls_established', index=4,
number=5, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='cert', full_name='ServerConnection.cert', index=5,
number=6, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='sni', full_name='ServerConnection.sni', index=6,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='alpn_proto_negotiated', full_name='ServerConnection.alpn_proto_negotiated', index=7,
number=8, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tls_version', full_name='ServerConnection.tls_version', index=8,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_start', full_name='ServerConnection.timestamp_start', index=9,
number=10, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_tcp_setup', full_name='ServerConnection.timestamp_tcp_setup', index=10,
number=11, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_tls_setup', full_name='ServerConnection.timestamp_tls_setup', index=11,
number=12, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp_end', full_name='ServerConnection.timestamp_end', index=12,
number=13, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='via', full_name='ServerConnection.via', index=13,
number=14, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=1157,
serialized_end=1520,
)
_TLSEXTENSION = _descriptor.Descriptor(
name='TLSExtension',
full_name='TLSExtension',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='int', full_name='TLSExtension.int', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='bytes', full_name='TLSExtension.bytes', index=1,
number=2, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=1522,
serialized_end=1564,
)
_HTTPFLOW.fields_by_name['request'].message_type = _HTTPREQUEST
_HTTPFLOW.fields_by_name['response'].message_type = _HTTPRESPONSE
_HTTPFLOW.fields_by_name['error'].message_type = _HTTPERROR
_HTTPFLOW.fields_by_name['client_conn'].message_type = _CLIENTCONNECTION
_HTTPFLOW.fields_by_name['server_conn'].message_type = _SERVERCONNECTION
_HTTPREQUEST.fields_by_name['headers'].message_type = _HTTPHEADER
_HTTPRESPONSE.fields_by_name['headers'].message_type = _HTTPHEADER
_CLIENTCONNECTION.fields_by_name['address'].message_type = _ADDRESS
_CLIENTCONNECTION.fields_by_name['tls_extensions'].message_type = _TLSEXTENSION
_SERVERCONNECTION.fields_by_name['address'].message_type = _ADDRESS
_SERVERCONNECTION.fields_by_name['ip_address'].message_type = _ADDRESS
_SERVERCONNECTION.fields_by_name['source_address'].message_type = _ADDRESS
_SERVERCONNECTION.fields_by_name['via'].message_type = _SERVERCONNECTION
DESCRIPTOR.message_types_by_name['HTTPFlow'] = _HTTPFLOW
DESCRIPTOR.message_types_by_name['HTTPRequest'] = _HTTPREQUEST
DESCRIPTOR.message_types_by_name['HTTPResponse'] = _HTTPRESPONSE
DESCRIPTOR.message_types_by_name['HTTPError'] = _HTTPERROR
DESCRIPTOR.message_types_by_name['HTTPHeader'] = _HTTPHEADER
DESCRIPTOR.message_types_by_name['Address'] = _ADDRESS
DESCRIPTOR.message_types_by_name['ClientConnection'] = _CLIENTCONNECTION
DESCRIPTOR.message_types_by_name['ServerConnection'] = _SERVERCONNECTION
DESCRIPTOR.message_types_by_name['TLSExtension'] = _TLSEXTENSION
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
HTTPFlow = _reflection.GeneratedProtocolMessageType('HTTPFlow', (_message.Message,), dict(
DESCRIPTOR = _HTTPFLOW,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:HTTPFlow)
))
_sym_db.RegisterMessage(HTTPFlow)
HTTPRequest = _reflection.GeneratedProtocolMessageType('HTTPRequest', (_message.Message,), dict(
DESCRIPTOR = _HTTPREQUEST,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:HTTPRequest)
))
_sym_db.RegisterMessage(HTTPRequest)
HTTPResponse = _reflection.GeneratedProtocolMessageType('HTTPResponse', (_message.Message,), dict(
DESCRIPTOR = _HTTPRESPONSE,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:HTTPResponse)
))
_sym_db.RegisterMessage(HTTPResponse)
HTTPError = _reflection.GeneratedProtocolMessageType('HTTPError', (_message.Message,), dict(
DESCRIPTOR = _HTTPERROR,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:HTTPError)
))
_sym_db.RegisterMessage(HTTPError)
HTTPHeader = _reflection.GeneratedProtocolMessageType('HTTPHeader', (_message.Message,), dict(
DESCRIPTOR = _HTTPHEADER,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:HTTPHeader)
))
_sym_db.RegisterMessage(HTTPHeader)
Address = _reflection.GeneratedProtocolMessageType('Address', (_message.Message,), dict(
DESCRIPTOR = _ADDRESS,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:Address)
))
_sym_db.RegisterMessage(Address)
ClientConnection = _reflection.GeneratedProtocolMessageType('ClientConnection', (_message.Message,), dict(
DESCRIPTOR = _CLIENTCONNECTION,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:ClientConnection)
))
_sym_db.RegisterMessage(ClientConnection)
ServerConnection = _reflection.GeneratedProtocolMessageType('ServerConnection', (_message.Message,), dict(
DESCRIPTOR = _SERVERCONNECTION,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:ServerConnection)
))
_sym_db.RegisterMessage(ServerConnection)
TLSExtension = _reflection.GeneratedProtocolMessageType('TLSExtension', (_message.Message,), dict(
DESCRIPTOR = _TLSEXTENSION,
__module__ = 'http_pb2'
# @@protoc_insertion_point(class_scope:TLSExtension)
))
_sym_db.RegisterMessage(TLSExtension)
# @@protoc_insertion_point(module_scope)

198
mitmproxy/io/protobuf.py Normal file
View File

@ -0,0 +1,198 @@
import typing
from mitmproxy import flow
from mitmproxy import exceptions
from mitmproxy.http import HTTPFlow, HTTPResponse, HTTPRequest
from mitmproxy.certs import Cert
from mitmproxy.connections import ClientConnection, ServerConnection
from mitmproxy.io.proto import http_pb2
def _move_attrs(s_obj, d_obj, attrs):
for attr in attrs:
if not isinstance(d_obj, dict):
if hasattr(s_obj, attr) and getattr(s_obj, attr) is not None:
setattr(d_obj, attr, getattr(s_obj, attr))
else:
if hasattr(s_obj, attr) and getattr(s_obj, attr) is not None:
# ugly fix to set None in empty str or bytes fields
if getattr(s_obj, attr) == "" or getattr(s_obj, attr) == b"":
d_obj[attr] = None
else:
d_obj[attr] = getattr(s_obj, attr)
def _dump_http_response(res: HTTPResponse) -> http_pb2.HTTPResponse:
pres = http_pb2.HTTPResponse()
_move_attrs(res, pres, ['http_version', 'status_code', 'reason',
'content', 'timestamp_start', 'timestamp_end', 'is_replay'])
if res.headers:
for h in res.headers.fields:
header = pres.headers.add()
header.name = h[0]
header.value = h[1]
return pres
def _dump_http_request(req: HTTPRequest) -> http_pb2.HTTPRequest:
preq = http_pb2.HTTPRequest()
_move_attrs(req, preq, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content',
'timestamp_start', 'timestamp_end', 'is_replay'])
if req.headers:
for h in req.headers.fields:
header = preq.headers.add()
header.name = h[0]
header.value = h[1]
return preq
def _dump_http_client_conn(cc: ClientConnection) -> http_pb2.ClientConnection:
pcc = http_pb2.ClientConnection()
_move_attrs(cc, pcc, ['id', 'tls_established', 'timestamp_start', 'timestamp_tls_setup', 'timestamp_end', 'sni',
'cipher_name', 'alpn_proto_negotiated', 'tls_version'])
for cert in ['clientcert', 'mitmcert']:
if hasattr(cc, cert) and getattr(cc, cert) is not None:
setattr(pcc, cert, getattr(cc, cert).to_pem())
if cc.tls_extensions:
for extension in cc.tls_extensions:
ext = pcc.tls_extensions.add()
ext.int = extension[0]
ext.bytes = extension[1]
if cc.address:
pcc.address.host = cc.address[0]
pcc.address.port = cc.address[1]
return pcc
def _dump_http_server_conn(sc: ServerConnection) -> http_pb2.ServerConnection:
psc = http_pb2.ServerConnection()
_move_attrs(sc, psc, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version',
'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end'])
for addr in ['address', 'ip_address', 'source_address']:
if hasattr(sc, addr) and getattr(sc, addr) is not None:
getattr(psc, addr).host = getattr(sc, addr)[0]
getattr(psc, addr).port = getattr(sc, addr)[1]
if sc.cert:
psc.cert = sc.cert.to_pem()
if sc.via:
psc.via.MergeFrom(_dump_http_server_conn(sc.via))
return psc
def _dump_http_error(e: flow.Error) -> http_pb2.HTTPError:
pe = http_pb2.HTTPError()
for attr in ['msg', 'timestamp']:
if hasattr(e, attr) and getattr(e, attr) is not None:
setattr(pe, attr, getattr(e, attr))
return pe
def dump_http(f: flow.Flow) -> http_pb2.HTTPFlow:
pf = http_pb2.HTTPFlow()
for p in ['request', 'response', 'client_conn', 'server_conn', 'error']:
if hasattr(f, p) and getattr(f, p):
getattr(pf, p).MergeFrom(eval(f"_dump_http_{p}")(getattr(f, p)))
_move_attrs(f, pf, ['intercepted', 'marked', 'mode', 'id'])
return pf
def dumps(f: flow.Flow) -> bytes:
if f.type != "http":
raise exceptions.TypeError("Flow types different than HTTP not supported yet!")
else:
p = dump_http(f)
return p.SerializeToString()
def _load_http_request(o: http_pb2.HTTPRequest) -> HTTPRequest:
d: dict = {}
_move_attrs(o, d, ['first_line_format', 'method', 'scheme', 'host', 'port', 'path', 'http_version', 'content',
'timestamp_start', 'timestamp_end', 'is_replay'])
if d['content'] is None:
d['content'] = b""
d["headers"] = []
for header in o.headers:
d["headers"].append((bytes(header.name, "utf-8"), bytes(header.value, "utf-8")))
return HTTPRequest(**d)
def _load_http_response(o: http_pb2.HTTPResponse) -> HTTPResponse:
d: dict = {}
_move_attrs(o, d, ['http_version', 'status_code', 'reason',
'content', 'timestamp_start', 'timestamp_end', 'is_replay'])
if d['content'] is None:
d['content'] = b""
d["headers"] = []
for header in o.headers:
d["headers"].append((bytes(header.name, "utf-8"), bytes(header.value, "utf-8")))
return HTTPResponse(**d)
def _load_http_client_conn(o: http_pb2.ClientConnection) -> ClientConnection:
d: dict = {}
_move_attrs(o, d, ['id', 'tls_established', 'sni', 'cipher_name', 'alpn_proto_negotiated', 'tls_version',
'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end'])
for cert in ['clientcert', 'mitmcert']:
if hasattr(o, cert) and getattr(o, cert):
d[cert] = Cert.from_pem(getattr(o, cert))
if o.tls_extensions:
d['tls_extensions'] = []
for extension in o.tls_extensions:
d['tls_extensions'].append((extension.int, extension.bytes))
if o.address:
d['address'] = (o.address.host, o.address.port)
cc = ClientConnection(None, tuple(), None)
for k, v in d.items():
setattr(cc, k, v)
return cc
def _load_http_server_conn(o: http_pb2.ServerConnection) -> ServerConnection:
d: dict = {}
_move_attrs(o, d, ['id', 'tls_established', 'sni', 'alpn_proto_negotiated', 'tls_version',
'timestamp_start', 'timestamp_tcp_setup', 'timestamp_tls_setup', 'timestamp_end'])
for addr in ['address', 'ip_address', 'source_address']:
if hasattr(o, addr):
d[addr] = (getattr(o, addr).host, getattr(o, addr).port)
if o.cert:
c = Cert.from_pem(o.cert)
d['cert'] = c
if o.HasField('via'):
d['via'] = _load_http_server_conn(o.via)
sc = ServerConnection(tuple())
for k, v in d.items():
setattr(sc, k, v)
return sc
def _load_http_error(o: http_pb2.HTTPError) -> typing.Optional[flow.Error]:
d = {}
for m in ['msg', 'timestamp']:
if hasattr(o, m) and getattr(o, m):
d[m] = getattr(o, m)
return None if not d else flow.Error(**d)
def load_http(hf: http_pb2.HTTPFlow) -> HTTPFlow:
parts = {}
for p in ['request', 'response', 'client_conn', 'server_conn', 'error']:
if hf.HasField(p):
parts[p] = eval(f"_load_http_{p}")(getattr(hf, p))
else:
parts[p] = None
_move_attrs(hf, parts, ['intercepted', 'marked', 'mode', 'id'])
f = HTTPFlow(ClientConnection(None, tuple(), None), ServerConnection(tuple()))
for k, v in parts.items():
setattr(f, k, v)
return f
def loads(b: bytes, typ="http") -> flow.Flow:
if typ != 'http':
raise exceptions.TypeError("Flow types different than HTTP not supported yet!")
else:
p = http_pb2.HTTPFlow()
p.ParseFromString(b)
return load_http(p)

2
setup.cfg
View File

@ -2,7 +2,7 @@
max-line-length = 140 max-line-length = 140
max-complexity = 25 max-complexity = 25
ignore = E251,C901,W503,W292,E722,E741 ignore = E251,C901,W503,W292,E722,E741
exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/* exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/*,mitmproxy/io/proto/*
addons = file,open,basestring,xrange,unicode,long,cmp addons = file,open,basestring,xrange,unicode,long,cmp
[tool:pytest] [tool:pytest]

1
setup.py
View File

@ -70,6 +70,7 @@ setup(
"kaitaistruct>=0.7,<0.9", "kaitaistruct>=0.7,<0.9",
"ldap3>=2.5,<2.6", "ldap3>=2.5,<2.6",
"passlib>=1.6.5, <1.8", "passlib>=1.6.5, <1.8",
"protobuf>=3.6.0, <3.7",
"pyasn1>=0.3.1,<0.5", "pyasn1>=0.3.1,<0.5",
"pyOpenSSL>=17.5,<18.1", "pyOpenSSL>=17.5,<18.1",
"pyparsing>=2.1.3, <2.3", "pyparsing>=2.1.3, <2.3",

3
test/filename_matching.py
View File

@ -9,7 +9,8 @@ import sys
def check_src_files_have_test(): def check_src_files_have_test():
missing_test_files = [] missing_test_files = []
excluded = ['mitmproxy/contrib/', 'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/'] excluded = ['mitmproxy/contrib/', 'mitmproxy/io/proto/',
'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/']
src_files = glob.glob('mitmproxy/**/*.py', recursive=True) + glob.glob('pathod/**/*.py', recursive=True) src_files = glob.glob('mitmproxy/**/*.py', recursive=True) + glob.glob('pathod/**/*.py', recursive=True)
src_files = [f for f in src_files if os.path.basename(f) != '__init__.py'] src_files = [f for f in src_files if os.path.basename(f) != '__init__.py']
src_files = [f for f in src_files if not any(os.path.normpath(p) in f for p in excluded)] src_files = [f for f in src_files if not any(os.path.normpath(p) in f for p in excluded)]

26
test/mitmproxy/io/test_db.py Normal file
View File

@ -0,0 +1,26 @@
from mitmproxy.io import db
from mitmproxy.test import tflow
class TestDB:
def test_create(self, tdata):
dh = db.DBHandler(db_path=tdata.path("mitmproxy/data") + "/tmp.sqlite")
with dh._con as c:
cur = c.cursor()
cur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='FLOWS';")
assert cur.fetchall() == [('FLOWS',)]
def test_roundtrip(self, tdata):
dh = db.DBHandler(db_path=tdata.path("mitmproxy/data") + "/tmp.sqlite", mode='write')
flows = []
for i in range(10):
flows.append(tflow.tflow())
dh.store(flows)
dh = db.DBHandler(db_path=tdata.path("mitmproxy/data") + "/tmp.sqlite")
with dh._con as c:
cur = c.cursor()
cur.execute("SELECT count(*) FROM FLOWS;")
assert cur.fetchall()[0][0] == 10
loaded_flows = dh.load()
assert len(loaded_flows) == len(flows)

120
test/mitmproxy/io/test_protobuf.py Normal file
View File

@ -0,0 +1,120 @@
import pytest
from mitmproxy import certs
from mitmproxy import http
from mitmproxy import exceptions
from mitmproxy.test import tflow, tutils
from mitmproxy.io import protobuf
class TestProtobuf:
def test_roundtrip_client(self):
c = tflow.tclient_conn()
del c.reply
c.rfile = None
c.wfile = None
pc = protobuf._dump_http_client_conn(c)
lc = protobuf._load_http_client_conn(pc)
assert c.__dict__ == lc.__dict__
def test_roundtrip_client_cert(self, tdata):
c = tflow.tclient_conn()
c.rfile = None
c.wfile = None
del c.reply
with open(tdata.path("mitmproxy/net/data/clientcert/client.pem"), "rb") as f:
d = f.read()
c.clientcert = certs.Cert.from_pem(d)
pc = protobuf._dump_http_client_conn(c)
lc = protobuf._load_http_client_conn(pc)
assert c.__dict__ == lc.__dict__
def test_roundtrip_server(self):
s = tflow.tserver_conn()
del s.reply
s.wfile = None
s.rfile = None
ps = protobuf._dump_http_server_conn(s)
ls = protobuf._load_http_server_conn(ps)
assert s.__dict__ == ls.__dict__
def test_roundtrip_server_cert(self, tdata):
s = tflow.tserver_conn()
del s.reply
s.wfile = None
s.rfile = None
with open(tdata.path("mitmproxy/net/data/text_cert"), "rb") as f:
d = f.read()
s.cert = certs.Cert.from_pem(d)
ps = protobuf._dump_http_server_conn(s)
ls = protobuf._load_http_server_conn(ps)
assert s.__dict__ == ls.__dict__
def test_roundtrip_server_via(self):
s = tflow.tserver_conn()
s.via = tflow.tserver_conn()
del s.reply
s.wfile = None
s.rfile = None
ps = protobuf._dump_http_server_conn(s)
ls = protobuf._load_http_server_conn(ps)
assert s.__dict__ == ls.__dict__
del s.via.reply
s.via.wfile = None
s.via.rfile = None
assert s.via.__dict__ == ls.via.__dict__
def test_roundtrip_http_request(self):
req = http.HTTPRequest.wrap(tutils.treq())
preq = protobuf._dump_http_request(req)
lreq = protobuf._load_http_request(preq)
assert req.__dict__ == lreq.__dict__
def test_roundtrip_http_request_empty_content(self):
req = http.HTTPRequest.wrap(tutils.treq(content=b""))
preq = protobuf._dump_http_request(req)
lreq = protobuf._load_http_request(preq)
assert req.__dict__ == lreq.__dict__
def test_roundtrip_http_response(self):
res = http.HTTPResponse.wrap(tutils.tresp())
pres = protobuf._dump_http_response(res)
lres = protobuf._load_http_response(pres)
assert res.__dict__ == lres.__dict__
def test_roundtrip_http_response_empty_content(self):
res = http.HTTPResponse.wrap(tutils.tresp(content=b""))
pres = protobuf._dump_http_response(res)
lres = protobuf._load_http_response(pres)
assert res.__dict__ == lres.__dict__
def test_roundtrip_http_error(self):
err = tflow.terr()
perr = protobuf._dump_http_error(err)
lerr = protobuf._load_http_error(perr)
assert err.__dict__ == lerr.__dict__
def test_roundtrip_http_flow_only_req(self):
f = tflow.tflow()
f.reply = None
pf = protobuf.dumps(f)
lf = protobuf.loads(pf, "http")
assert f.__dict__ == lf.__dict__
def test_roundtrip_http_flow_res(self):
f = tflow.tflow(resp=True)
f.reply = None
pf = protobuf.dumps(f)
lf = protobuf.loads(pf, "http")
assert f.__dict__ == lf.__dict__
def test_unsupported_dumps(self):
w = tflow.twebsocketflow()
with pytest.raises(exceptions.TypeError):
protobuf.dumps(w)
def test_unsupported_loads(self):
b = b"blobs"
with pytest.raises(exceptions.TypeError):
protobuf.loads(b, 'not-http')