Merge pull request #4413 from r00t-/compat_convert_fix

fix conversion of old flows failing due to missing array keys

.gitattributes

@@ -1,3 +1,4 @@
 mitmproxy/tools/web/static/**/* -diff linguist-vendored
 web/src/js/filt/filt.js -diff
 *.bin binary
+*.mitm binary

mitmproxy/io/compat.py

@@ -174,19 +174,22 @@ def convert_6_7(data):
 
 def convert_7_8(data):
     data["version"] = 8
-    data["request"]["trailers"] = None
+    if "request" in data and data["request"] is not None:
-    if data["response"] is not None:
+        data["request"]["trailers"] = None
+    if "response" in data and data["response"] is not None:
         data["response"]["trailers"] = None
     return data
 
 
 def convert_8_9(data):
     data["version"] = 9
-    data["request"].pop("first_line_format")
+    is_request_replay = False
-    data["request"]["authority"] = b""
+    if "request" in data:
-    is_request_replay = data["request"].pop("is_replay", False)
+        data["request"].pop("first_line_format")
+        data["request"]["authority"] = b""
+        is_request_replay = data["request"].pop("is_replay", False)
     is_response_replay = False
-    if data["response"] is not None:
+    if "response" in data and data["response"] is not None:
         is_response_replay = data["response"].pop("is_replay", False)
     if is_request_replay:  # pragma: no cover
         data["is_replay"] = "request"

test/mitmproxy/addons/test_clientplayback.py

@@ -120,7 +120,7 @@ async def test_start_stop(tdata):
 def test_load(tdata):
     cp = ClientPlayback()
     with taddons.context(cp):
-        cp.load_file(tdata.path("mitmproxy/data/dumpfile-018.bin"))
+        cp.load_file(tdata.path("mitmproxy/data/dumpfile-018.mitm"))
         assert cp.count() == 1
 
         with pytest.raises(CommandError):
@@ -132,7 +132,7 @@ def test_configure(tdata):
     cp = ClientPlayback()
     with taddons.context(cp) as tctx:
         assert cp.count() == 0
-        tctx.configure(cp, client_replay=[tdata.path("mitmproxy/data/dumpfile-018.bin")])
+        tctx.configure(cp, client_replay=[tdata.path("mitmproxy/data/dumpfile-018.mitm")])
         assert cp.count() == 1
         tctx.configure(cp, client_replay=[])
         with pytest.raises(OptionsError):

test/mitmproxy/io/test_compat.py

@@ -4,21 +4,22 @@ from mitmproxy import io
 from mitmproxy import exceptions
 
 
-@pytest.mark.parametrize("dumpfile, url", [
+@pytest.mark.parametrize("dumpfile, url, count", [
-    ["dumpfile-011.bin", "https://example.com/"],
+    ["dumpfile-011.mitm", "https://example.com/", 1],
-    ["dumpfile-018.bin", "https://www.example.com/"],
+    ["dumpfile-018.mitm", "https://www.example.com/", 1],
-    ["dumpfile-019.bin", "https://webrv.rtb-seller.com/"],
+    ["dumpfile-019.mitm", "https://webrv.rtb-seller.com/", 1],
+    ["dumpfile-7-websocket.mitm", "https://echo.websocket.org/", 5],
 ])
-def test_load(tdata, dumpfile, url):
+def test_load(tdata, dumpfile, url, count):
     with open(tdata.path("mitmproxy/data/" + dumpfile), "rb") as f:
         flow_reader = io.FlowReader(f)
         flows = list(flow_reader.stream())
-        assert len(flows) == 1
+        assert len(flows) == count
-        assert flows[0].request.url.startswith(url)
+        assert flows[-1].request.url.startswith(url)
 
 
 def test_cannot_convert(tdata):
-    with open(tdata.path("mitmproxy/data/dumpfile-010.bin"), "rb") as f:
+    with open(tdata.path("mitmproxy/data/dumpfile-010.mitm"), "rb") as f:
         flow_reader = io.FlowReader(f)
         with pytest.raises(exceptions.FlowReadException):
             list(flow_reader.stream())

test/mitmproxy/tools/console/test_integration.py

@@ -40,6 +40,6 @@ def console(monkeypatch):
 
 
 def test_integration(tdata, console):
-    console.type(f":view.flows.load {tdata.path('mitmproxy/data/dumpfile-7.bin')}<enter>")
+    console.type(f":view.flows.load {tdata.path('mitmproxy/data/dumpfile-7.mitm')}<enter>")
     console.type("<enter><tab><tab>")
     console.type("<space><tab><tab>")  # view second flow