Grasscutters/mitmproxy
3
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-22 07:08:10 +00:00
Code Issues Projects Releases Wiki Activity

gRPC view: Adjusted parsing strategy for deprecated protobuf fields (#4858)

Browse Source 
* Adjusted parsing strategy for deprecated protobuf fields

* Added test comment

* adjusted comment to reflect correct field IDs
This commit is contained in:
mame82 2021-10-13 13:16:49 +02:00 committed by GitHub
parent 77cf2ab4ee
commit 8b6345e4a7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 78 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mitmproxy/contentviews/grpc.py
View File

@ -172,8 +172,8 @@ class ProtoParser:
tag = pair.field_tag
wt = pair.wire_type
if wt == GoogleProtobuf.Pair.WireTypes.group_start or wt == GoogleProtobuf.Pair.WireTypes.group_end:
# ignore deprecated types without values
continue
# raise error on deprecated types without values
raise ValueError("deprecated field: {}".format(wt))
v: Union[GoogleProtobuf.DelimitedBytes, VlqBase128Le] = pair.value # for WireType bit-32 and bit-64
preferred_decoding = ProtoParser.DecodedTypes.unknown
# see: https://www.oreilly.com/library/view/grpc-up-and/9781492058328/ch04.html

77
test/mitmproxy/contentviews/test_grpc.py
View File

@ -110,6 +110,13 @@ def test_view_protobuf_custom_parsing_request(tdata):
def test_view_protobuf_custom_parsing_response(tdata):
# expect to parse 1.3.2 and 1.3.3 as string automatically
# even if there is a length delimeted field containing `b"DC"`, which would translate to
# two deprecated fields [8: group_start, 8: group_end] (and thus represent a valid nested message,
# but containing deprecated wire types)
custom_view_config_parser_rules.parser_rules[1].field_definitions[3].intended_decoding = None
custom_view_config_parser_rules.parser_rules[1].field_definitions[4].intended_decoding = None
v = full_eval(ViewGrpcProtobuf(custom_view_config_parser_rules))
p = tdata.path(datadir + "msg3.bin")
@ -141,7 +148,75 @@ def test_view_protobuf_custom_parsing_response(tdata):
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x06\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'District of Columbia ')], # noqa: E501
[('text', '[message] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', ' ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'DC ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x05\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'USA ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'US ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x17' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', '20502 ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', '20502 ')], # noqa: E501
[('text', '[message] '), ('text', ' '), ('text', '1.5 '), ('text', ' ')], # noqa: E501
[('text', '[message] '), ('text', 'position '), ('text', '1.5.1 '), ('text', ' ')], # noqa: E501
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.1.1 '), ('text', '38.8970309 ')], # noqa: E501
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.1.2 '), ('text', '-77.03872559999999 ')], # noqa: E501
[('text', '[uint32] '), ('text', ' '), ('text', '1.5.2 '), ('text', '1 ')], # noqa: E501
[('text', '[message] '), ('text', 'position '), ('text', '1.5.3 '), ('text', ' ')], # noqa: E501
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.3.1 '), ('text', '38.8962271697085 ')], # noqa: E501
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.3.2 '), ('text', '-77.0400511802915 ')], # noqa: E501
[('text', '[message] '), ('text', 'position '), ('text', '1.5.4 '), ('text', ' ')], # noqa: E501
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.4.1 '), ('text', '38.8989251302915 ')], # noqa: E501
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.4.2 '), ('text', '-77.03735321970849 ')], # noqa: E501
[('text', '[message] '), ('text', 'position '), ('text', '1.5.5 '), ('text', ' ')], # noqa: E501
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.5.1 '), ('text', '38.896898 ')], # noqa: E501
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.5.2 '), ('text', '-77.03917229999999 ')], # noqa: E501
[('text', '[message] '), ('text', 'position '), ('text', '1.5.6 '), ('text', ' ')], # noqa: E501
[('text', '[double] '), ('text', 'latitude '), ('text', '1.5.6.1 '), ('text', '38.8982543 ')], # noqa: E501
[('text', '[double] '), ('text', 'longitude '), ('text', '1.5.6.2 '), ('text', '-77.0382321 ')], # noqa: E501
[('text', '[string] '), ('text', ' '), ('text', '1.7 '), ('text', 'ChIJAXiAory3t4kRpkrvas9dYmQ ')], # noqa: E501
[('text', '[message] '), ('text', ' '), ('text', '2 '), ('text', ' ')], # noqa: E501
[('text', '[uint32] '), ('text', ' '), ('text', '2.1 '), ('text', '21 ')], # noqa: E501
]
def test_view_protobuf_custom_parsing_response2(tdata):
# try to parse 1.3.2 and 1.3.3 as string
custom_view_config_parser_rules.parser_rules[1].field_definitions[3].intended_decoding = ProtoParser.DecodedTypes.string # 1.3.2
custom_view_config_parser_rules.parser_rules[1].field_definitions[4].intended_decoding = ProtoParser.DecodedTypes.string # 1.3.3
v = full_eval(ViewGrpcProtobuf(custom_view_config_parser_rules))
p = tdata.path(datadir + "msg3.bin")
with open(p, "rb") as f:
raw = f.read()
view_text, output = v(raw, flow=sim_flow, http_message=sim_flow.response) # simulate response message
assert view_text == "Protobuf (flattened)"
output = list(output) # assure list conversion if generator
assert output == [
[('text', '[message] '), ('text', ' '), ('text', '1 '), ('text', ' ')], # noqa: E501
[('text', '[string] '), ('text', ' '), ('text', '1.1 '), ('text', '\x15 ')], # noqa: E501
[('text', '[string] '), ('text', 'address '), ('text', '1.2 '), ('text', '1650 Pennsylvania Avenue NW, Washington, DC 20502, USA ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', 'b\'"\' ')], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', '1650 ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', '1650 ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x02' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'Pennsylvania Avenue Northwest ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'Pennsylvania Avenue NW ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x14\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'Northwest Washington ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'Northwest Washington ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x0c\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'Washington ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'Washington ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x06\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'District of Columbia ')], # noqa: E501
[('text', '[string] '), ('text', 'element value short '), ('text', '1.3.3 '), ('text', 'DC ')], # noqa: E501
[('text', '[message] '), ('text', 'address array element '), ('text', '1.3 '), ('text', ' ')], # noqa: E501
[('text', '[bytes] '), ('text', 'unknown bytes '), ('text', '1.3.1 '), ('text', "b'\\x05\\x04' ")], # noqa: E501
[('text', '[string] '), ('text', 'element value long '), ('text', '1.3.2 '), ('text', 'USA ')], # noqa: E501