add Flow.timestamp_created, which retains list order on replay (#5227)

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased: mitmproxy next
 
+* Replayed flows retain their current position in the flow list.
+  ([#5227](https://github.com/mitmproxy/mitmproxy/issues/5227), @mhils)
 * Console Performance Improvements
   ([#3427](https://github.com/mitmproxy/mitmproxy/issues/3427), @BkPHcgQL3V)
 * Add flatpak support to the browser addon

mitmproxy/addons/view.py

@@ -74,7 +74,7 @@ class _OrderKey:
 
 class OrderRequestStart(_OrderKey):
     def generate(self, f: mitmproxy.flow.Flow) -> float:
-        return f.timestamp_start
+        return f.timestamp_created
 
 
 class OrderRequestMethod(_OrderKey):

mitmproxy/flow.py

@@ -111,6 +111,13 @@ class Flow(stateobject.StateObject):
     If `False`, the flow may have been already completed or loaded from disk.
     """
 
+    timestamp_created: float
+    """
+    The Unix timestamp of when this flow was created.
+
+    In contrast to `timestamp_start`, this value will not change when a flow is replayed.
+    """
+
     def __init__(
         self,
         type: str,
@@ -123,6 +130,7 @@ class Flow(stateobject.StateObject):
         self.client_conn = client_conn
         self.server_conn = server_conn
         self.live = live
+        self.timestamp_created = time.time()
 
         self.intercepted: bool = False
         self._resume_event: typing.Optional[asyncio.Event] = None
@@ -143,6 +151,7 @@ class Flow(stateobject.StateObject):
         marked=str,
         metadata=typing.Dict[str, typing.Any],
         comment=str,
+        timestamp_created=float,
     )
 
     def get_state(self):

mitmproxy/io/compat.py

@@ -330,6 +330,12 @@ def convert_14_15(data):
     return data
 
 
+def convert_15_16(data):
+    data["version"] = 16
+    data["timestamp_created"] = data.get("request", data["client_conn"])["timestamp_start"]
+    return data
+
+
 def _convert_dict_keys(o: Any) -> Any:
     if isinstance(o, dict):
         return {strutils.always_str(k): _convert_dict_keys(v) for k, v in o.items()}
@@ -392,6 +398,7 @@ converters = {
     12: convert_12_13,
     13: convert_13_14,
     14: convert_14_15,
+    15: convert_15_16,
 }
 
 

mitmproxy/test/tflow.py

@@ -24,6 +24,7 @@ def ttcpflow(client_conn=True, server_conn=True, messages=True, err=None) -> tcp
         err = terr()
 
     f = tcp.TCPFlow(client_conn, server_conn)
+    f.timestamp_created = client_conn.timestamp_start
     f.messages = messages
     f.error = err
     f.live = True
@@ -115,6 +116,7 @@ def tflow(
     assert ws is False or isinstance(ws, websocket.WebSocketData)
 
     f = http.HTTPFlow(client_conn, server_conn)
+    f.timestamp_created = req.timestamp_start
     f.request = req
     f.response = resp or None
     f.error = err or None

mitmproxy/tools/console/common.py

@@ -669,13 +669,13 @@ def format_flow(
         for message in f.messages:
             total_size += len(message.content)
         if f.messages:
-            duration = f.messages[-1].timestamp - f.timestamp_start
+            duration = f.messages[-1].timestamp - f.client_conn.timestamp_start
         else:
             duration = None
         return format_tcp_flow(
             render_mode=render_mode,
             focused=focused,
-            timestamp_start=f.timestamp_start,
+            timestamp_start=f.client_conn.timestamp_start,
             marked=f.marked,
             client_address=f.client_conn.peername,
             server_address=f.server_conn.address,

mitmproxy/tools/web/app.py

@@ -59,6 +59,7 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict:
         "modified": flow.modified(),
         "marked": emoji.get(flow.marked, "🔴") if flow.marked else "",
         "comment": flow.comment,
+        "timestamp_created": flow.timestamp_created,
     }
 
     if flow.client_conn:

mitmproxy/version.py

@@ -7,7 +7,7 @@ MITMPROXY = "mitmproxy " + VERSION
 
 # Serialization format version. This is displayed nowhere, it just needs to be incremented by one
 # for each change in the file format.
-FLOW_FORMAT_VERSION = 15
+FLOW_FORMAT_VERSION = 16
 
 
 def get_dev_version() -> str:

test/mitmproxy/addons/test_view.py

@@ -14,7 +14,7 @@ from mitmproxy.tools.console.common import render_marker, SYMBOL_MARK
 def tft(*, method="get", start=0):
     f = tflow.tflow()
     f.request.method = method
-    f.request.timestamp_start = start
+    f.timestamp_created = start
     return f
 
 
@@ -31,7 +31,7 @@ def test_order_refresh():
     with taddons.context() as tctx:
         tctx.configure(v, view_order="time")
         v.add([tf])
-        tf.request.timestamp_start = 10
+        tf.timestamp_created = 10
         assert not sargs
         v.update([tf])
         assert sargs
@@ -345,7 +345,7 @@ def test_order():
     v.requestheaders(tft(method="put", start=2))
     v.requestheaders(tft(method="get", start=3))
     v.requestheaders(tft(method="put", start=4))
-    assert [i.request.timestamp_start for i in v] == [1, 2, 3, 4]
+    assert [i.timestamp_created for i in v] == [1, 2, 3, 4]
 
     v.set_order("method")
     assert v.get_order() == "method"
@@ -355,10 +355,10 @@ def test_order():
 
     v.set_order("time")
     assert v.get_order() == "time"
-    assert [i.request.timestamp_start for i in v] == [4, 3, 2, 1]
+    assert [i.timestamp_created for i in v] == [4, 3, 2, 1]
 
     v.set_reversed(False)
-    assert [i.request.timestamp_start for i in v] == [1, 2, 3, 4]
+    assert [i.timestamp_created for i in v] == [1, 2, 3, 4]
     with pytest.raises(exceptions.CommandError):
         v.set_order("not_an_order")
 
@@ -370,9 +370,9 @@ def test_reversed():
     v.requestheaders(tft(start=3))
     v.set_reversed(True)
 
-    assert v[0].request.timestamp_start == 3
+    assert v[0].timestamp_created == 3
-    assert v[-1].request.timestamp_start == 1
+    assert v[-1].timestamp_created == 1
-    assert v[2].request.timestamp_start == 1
+    assert v[2].timestamp_created == 1
     with pytest.raises(IndexError):
         v[5]
     with pytest.raises(IndexError):
@@ -485,21 +485,21 @@ def test_focus_follow():
 
         v.add([tft(start=4)])
         assert v.focus.index == 0
-        assert v.focus.flow.request.timestamp_start == 4
+        assert v.focus.flow.timestamp_created == 4
 
         v.add([tft(start=7)])
         assert v.focus.index == 2
-        assert v.focus.flow.request.timestamp_start == 7
+        assert v.focus.flow.timestamp_created == 7
 
         mod = tft(method="put", start=6)
         v.add([mod])
         assert v.focus.index == 2
-        assert v.focus.flow.request.timestamp_start == 7
+        assert v.focus.flow.timestamp_created == 7
 
         mod.request.method = "GET"
         v.update([mod])
         assert v.focus.index == 2
-        assert v.focus.flow.request.timestamp_start == 6
+        assert v.focus.flow.timestamp_created == 6
 
 
 def test_focus():

web/src/js/__tests__/ducks/_tflow.ts

@@ -145,6 +145,7 @@ export function THTTPFlow(): Required<HTTPFlow> {
             "tls_established": true,
             "tls_version": "TLSv1.2"
         },
+        "timestamp_created": 946681200,
         "type": "http",
         "websocket": {
             "close_code": 1000,
@@ -221,6 +222,7 @@ export function TTCPFlow(): Required<TCPFlow> {
             "tls_established": true,
             "tls_version": "TLSv1.2"
         },
+        "timestamp_created": 946681200,
         "type": "tcp"
     }
 }

web/src/js/flow.ts

@@ -9,6 +9,7 @@ interface _Flow {
     modified: boolean
     marked: string
     comment: string
+    timestamp_created: number
     client_conn: Client
     server_conn?: Server
     error?: Error