From cd913d598da421b4c4e264e029be5f13e3e009bb Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 16 Dec 2017 10:30:08 +1300 Subject: [PATCH] command cuts: add completion - Remove shortcuts for request, response, etc. - we don't need them if we have completion - Restrict cuts specification to a set of prefixes - Extend cuts to add a few more items --- mitmproxy/addons/cut.py | 42 +++---- mitmproxy/command.py | 41 ++++++- mitmproxy/test/tflow.py | 4 + mitmproxy/test/tutils.py | 8 +- .../tools/console/commander/commander.py | 13 +++ mitmproxy/tools/console/defaultkeys.py | 2 +- test/mitmproxy/addons/test_cut.py | 107 +++++++++--------- test/mitmproxy/addons/test_view.py | 4 +- test/mitmproxy/net/http/test_response.py | 4 +- web/src/js/__tests__/ducks/_tflow.js | 4 +- 10 files changed, 141 insertions(+), 88 deletions(-) diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py index 6b9dc7235..efc9e5df4 100644 --- a/mitmproxy/addons/cut.py +++ b/mitmproxy/addons/cut.py @@ -17,14 +17,6 @@ def headername(spec: str): return spec[len("header["):-1].strip() -flow_shortcuts = { - "q": "request", - "s": "response", - "cc": "client_conn", - "sc": "server_conn", -} - - def is_addr(v): return isinstance(v, tuple) and len(v) > 1 @@ -35,8 +27,6 @@ def extract(cut: str, f: flow.Flow) -> typing.Union[str, bytes]: for i, spec in enumerate(path): if spec.startswith("_"): raise exceptions.CommandError("Can't access internal attribute %s" % spec) - if isinstance(current, flow.Flow): - spec = flow_shortcuts.get(spec, spec) part = getattr(current, spec, None) if i == len(path) - 1: @@ -65,13 +55,12 @@ class Cut: ) -> command.Cuts: """ Cut data from a set of flows. Cut specifications are attribute paths - from the base of the flow object, with a few conveniences - "q", - "s", "cc" and "sc" are shortcuts for request, response, client_conn - and server_conn, "port" and "host" retrieve parts of an address - tuple, ".header[key]" retrieves a header value. Return values - converted to strings or bytes: SSL certicates are converted to PEM - format, bools are "true" or "false", "bytes" are preserved, and all - other values are converted to strings. + from the base of the flow object, with a few conveniences - "port" + and "host" retrieve parts of an address tuple, ".header[key]" + retrieves a header value. Return values converted to strings or + bytes: SSL certicates are converted to PEM format, bools are "true" + or "false", "bytes" are preserved, and all other values are + converted to strings. """ ret = [] for f in flows: @@ -101,11 +90,11 @@ class Cut: if fp.tell() > 0: # We're appending to a file that already exists and has content fp.write(b"\n") - for v in [extract(cuts[0], f) for f in flows]: - if isinstance(v, bytes): - fp.write(v) - else: - fp.write(v.encode("utf8")) + v = extract(cuts[0], flows[0]) + if isinstance(v, bytes): + fp.write(v) + else: + fp.write(v.encode("utf8")) ctx.log.alert("Saved single cut.") else: with open(path, "a" if append else "w", newline='', encoding="utf8") as fp: @@ -129,8 +118,8 @@ class Cut: column, the data is written to file as-is, with raw bytes preserved. """ fp = io.StringIO(newline="") - if len(cuts) == 1 and len(cuts[0]) == 1: - v = cuts[0][0] + if len(cuts) == 1 and len(flows) == 1: + v = extract(cuts[0], flows[0]) if isinstance(v, bytes): fp.write(strutils.always_str(v)) else: @@ -138,9 +127,10 @@ class Cut: ctx.log.alert("Clipped single cut.") else: writer = csv.writer(fp) - for r in cuts: + for f in flows: + vals = [extract(c, f) for c in cuts] writer.writerow( - [strutils.always_str(c) or "" for c in r] # type: ignore + [strutils.always_str(v) or "" for v in vals] # type: ignore ) ctx.log.alert("Clipped %s cuts as CSV." % len(cuts)) pyperclip.copy(fp.getvalue()) diff --git a/mitmproxy/command.py b/mitmproxy/command.py index 82bad4fa0..7d7fa7353 100644 --- a/mitmproxy/command.py +++ b/mitmproxy/command.py @@ -30,7 +30,46 @@ Cuts = typing.Sequence[ class Cut(str): - pass + # This is an awkward location for these values, but it's better than having + # the console core import and depend on an addon. FIXME: Add a way for + # addons to add custom types and manage their completion and validation. + valid_prefixes = [ + "request.method", + "request.scheme", + "request.host", + "request.http_version", + "request.port", + "request.path", + "request.url", + "request.text", + "request.content", + "request.raw_content", + "request.timestamp_start", + "request.timestamp_end", + "request.header[", + + "response.status_code", + "response.reason", + "response.text", + "response.content", + "response.timestamp_start", + "response.timestamp_end", + "response.raw_content", + "response.header[", + + "client_conn.address.port", + "client_conn.address.host", + "client_conn.tls_version", + "client_conn.sni", + "client_conn.ssl_established", + + "server_conn.address.port", + "server_conn.address.host", + "server_conn.ip_address.host", + "server_conn.tls_version", + "server_conn.sni", + "server_conn.ssl_established", + ] class Path(str): diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py index e754cb54e..c3dab30ce 100644 --- a/mitmproxy/test/tflow.py +++ b/mitmproxy/test/tflow.py @@ -53,6 +53,8 @@ def twebsocketflow(client_conn=True, server_conn=True, messages=True, err=None, sec_websocket_version="13", sec_websocket_key="1234", ), + timestamp_start=1, + timestamp_end=2, content=b'' ) resp = http.HTTPResponse( @@ -64,6 +66,8 @@ def twebsocketflow(client_conn=True, server_conn=True, messages=True, err=None, upgrade='websocket', sec_websocket_accept=b'', ), + timestamp_start=1, + timestamp_end=2, content=b'', ) handshake_flow = http.HTTPFlow(client_conn, server_conn) diff --git a/mitmproxy/test/tutils.py b/mitmproxy/test/tutils.py index 80e5b6fd7..bcce547ac 100644 --- a/mitmproxy/test/tutils.py +++ b/mitmproxy/test/tutils.py @@ -31,7 +31,9 @@ def treq(**kwargs): path=b"/path", http_version=b"HTTP/1.1", headers=http.Headers(((b"header", b"qvalue"), (b"content-length", b"7"))), - content=b"content" + content=b"content", + timestamp_start=1, + timestamp_end=2, ) default.update(kwargs) return http.Request(**default) @@ -48,8 +50,8 @@ def tresp(**kwargs): reason=b"OK", headers=http.Headers(((b"header-response", b"svalue"), (b"content-length", b"7"))), content=b"message", - timestamp_start=time.time(), - timestamp_end=time.time(), + timestamp_start=1, + timestamp_end=2, ) default.update(kwargs) return http.Response(**default) diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py index 5fc7dd128..b94d6f691 100644 --- a/mitmproxy/tools/console/commander/commander.py +++ b/mitmproxy/tools/console/commander/commander.py @@ -113,6 +113,19 @@ class CommandBuffer(): ), parse = parts, ) + if last.type == typing.Sequence[mitmproxy.command.Cut]: + spec = parts[-1].value.split(",") + opts = [] + for pref in mitmproxy.command.Cut.valid_prefixes: + spec[-1] = pref + opts.append(",".join(spec)) + self.completion = CompletionState( + completer = ListCompleter( + parts[-1].value, + opts, + ), + parse = parts, + ) elif isinstance(last.type, mitmproxy.command.Choice): self.completion = CompletionState( completer = ListCompleter( diff --git a/mitmproxy/tools/console/defaultkeys.py b/mitmproxy/tools/console/defaultkeys.py index b845a3ae6..7e078bbf7 100644 --- a/mitmproxy/tools/console/defaultkeys.py +++ b/mitmproxy/tools/console/defaultkeys.py @@ -31,7 +31,7 @@ def map(km): km.add("A", "flow.resume @all", ["flowlist", "flowview"], "Resume all intercepted flows") km.add("a", "flow.resume @focus", ["flowlist", "flowview"], "Resume this intercepted flow") km.add( - "b", "console.command cut.save @focus s.content ", + "b", "console.command cut.save @focus response.content ", ["flowlist", "flowview"], "Save response body to file" ) diff --git a/test/mitmproxy/addons/test_cut.py b/test/mitmproxy/addons/test_cut.py index bb3e1c2d2..0a523fff5 100644 --- a/test/mitmproxy/addons/test_cut.py +++ b/test/mitmproxy/addons/test_cut.py @@ -13,36 +13,41 @@ from unittest import mock def test_extract(): tf = tflow.tflow(resp=True) tests = [ - ["q.method", "GET"], - ["q.scheme", "http"], - ["q.host", "address"], - ["q.port", "22"], - ["q.path", "/path"], - ["q.url", "http://address:22/path"], - ["q.text", "content"], - ["q.content", b"content"], - ["q.raw_content", b"content"], - ["q.header[header]", "qvalue"], + ["request.method", "GET"], + ["request.scheme", "http"], + ["request.host", "address"], + ["request.http_version", "HTTP/1.1"], + ["request.port", "22"], + ["request.path", "/path"], + ["request.url", "http://address:22/path"], + ["request.text", "content"], + ["request.content", b"content"], + ["request.raw_content", b"content"], + ["request.timestamp_start", "1"], + ["request.timestamp_end", "2"], + ["request.header[header]", "qvalue"], - ["s.status_code", "200"], - ["s.reason", "OK"], - ["s.text", "message"], - ["s.content", b"message"], - ["s.raw_content", b"message"], - ["s.header[header-response]", "svalue"], + ["response.status_code", "200"], + ["response.reason", "OK"], + ["response.text", "message"], + ["response.content", b"message"], + ["response.raw_content", b"message"], + ["response.header[header-response]", "svalue"], + ["response.timestamp_start", "1"], + ["response.timestamp_end", "2"], - ["cc.address.port", "22"], - ["cc.address.host", "127.0.0.1"], - ["cc.tls_version", "TLSv1.2"], - ["cc.sni", "address"], - ["cc.ssl_established", "false"], + ["client_conn.address.port", "22"], + ["client_conn.address.host", "127.0.0.1"], + ["client_conn.tls_version", "TLSv1.2"], + ["client_conn.sni", "address"], + ["client_conn.ssl_established", "false"], - ["sc.address.port", "22"], - ["sc.address.host", "address"], - ["sc.ip_address.host", "192.168.0.1"], - ["sc.tls_version", "TLSv1.2"], - ["sc.sni", "address"], - ["sc.ssl_established", "false"], + ["server_conn.address.port", "22"], + ["server_conn.address.host", "address"], + ["server_conn.ip_address.host", "192.168.0.1"], + ["server_conn.tls_version", "TLSv1.2"], + ["server_conn.sni", "address"], + ["server_conn.ssl_established", "false"], ] for t in tests: ret = cut.extract(t[0], tf) @@ -53,7 +58,7 @@ def test_extract(): d = f.read() c1 = certs.SSLCert.from_pem(d) tf.server_conn.cert = c1 - assert "CERTIFICATE" in cut.extract("sc.cert", tf) + assert "CERTIFICATE" in cut.extract("server_conn.cert", tf) def test_headername(): @@ -74,15 +79,15 @@ def test_cut_clip(): v.add([tflow.tflow(resp=True)]) with mock.patch('pyperclip.copy') as pc: - tctx.command(c.clip, "@all", "q.method") + tctx.command(c.clip, "@all", "request.method") assert pc.called with mock.patch('pyperclip.copy') as pc: - tctx.command(c.clip, "@all", "q.content") + tctx.command(c.clip, "@all", "request.content") assert pc.called with mock.patch('pyperclip.copy') as pc: - tctx.command(c.clip, "@all", "q.method,q.content") + tctx.command(c.clip, "@all", "request.method,request.content") assert pc.called @@ -94,17 +99,17 @@ def test_cut_save(tmpdir): tctx.master.addons.add(v, c) v.add([tflow.tflow(resp=True)]) - tctx.command(c.save, "@all", "q.method", f) + tctx.command(c.save, "@all", "request.method", f) assert qr(f) == b"GET" - tctx.command(c.save, "@all", "q.content", f) + tctx.command(c.save, "@all", "request.content", f) assert qr(f) == b"content" - tctx.command(c.save, "@all", "q.content", "+" + f) + tctx.command(c.save, "@all", "request.content", "+" + f) assert qr(f) == b"content\ncontent" v.add([tflow.tflow(resp=True)]) - tctx.command(c.save, "@all", "q.method", f) + tctx.command(c.save, "@all", "request.method", f) assert qr(f).splitlines() == [b"GET", b"GET"] - tctx.command(c.save, "@all", "q.method,q.content", f) + tctx.command(c.save, "@all", "request.method,request.content", f) assert qr(f).splitlines() == [b"GET,content", b"GET,content"] @@ -112,20 +117,20 @@ def test_cut(): c = cut.Cut() with taddons.context(): tflows = [tflow.tflow(resp=True)] - assert c.cut(tflows, ["q.method"]) == [["GET"]] - assert c.cut(tflows, ["q.scheme"]) == [["http"]] - assert c.cut(tflows, ["q.host"]) == [["address"]] - assert c.cut(tflows, ["q.port"]) == [["22"]] - assert c.cut(tflows, ["q.path"]) == [["/path"]] - assert c.cut(tflows, ["q.url"]) == [["http://address:22/path"]] - assert c.cut(tflows, ["q.content"]) == [[b"content"]] - assert c.cut(tflows, ["q.header[header]"]) == [["qvalue"]] - assert c.cut(tflows, ["q.header[unknown]"]) == [[""]] + assert c.cut(tflows, ["request.method"]) == [["GET"]] + assert c.cut(tflows, ["request.scheme"]) == [["http"]] + assert c.cut(tflows, ["request.host"]) == [["address"]] + assert c.cut(tflows, ["request.port"]) == [["22"]] + assert c.cut(tflows, ["request.path"]) == [["/path"]] + assert c.cut(tflows, ["request.url"]) == [["http://address:22/path"]] + assert c.cut(tflows, ["request.content"]) == [[b"content"]] + assert c.cut(tflows, ["request.header[header]"]) == [["qvalue"]] + assert c.cut(tflows, ["request.header[unknown]"]) == [[""]] - assert c.cut(tflows, ["s.status_code"]) == [["200"]] - assert c.cut(tflows, ["s.reason"]) == [["OK"]] - assert c.cut(tflows, ["s.content"]) == [[b"message"]] - assert c.cut(tflows, ["s.header[header-response]"]) == [["svalue"]] + assert c.cut(tflows, ["response.status_code"]) == [["200"]] + assert c.cut(tflows, ["response.reason"]) == [["OK"]] + assert c.cut(tflows, ["response.content"]) == [[b"message"]] + assert c.cut(tflows, ["response.header[header-response]"]) == [["svalue"]] assert c.cut(tflows, ["moo"]) == [[""]] with pytest.raises(exceptions.CommandError): assert c.cut(tflows, ["__dict__"]) == [[""]] @@ -133,5 +138,5 @@ def test_cut(): c = cut.Cut() with taddons.context(): tflows = [tflow.ttcpflow()] - assert c.cut(tflows, ["q.method"]) == [[""]] - assert c.cut(tflows, ["s.status"]) == [[""]] + assert c.cut(tflows, ["request.method"]) == [[""]] + assert c.cut(tflows, ["response.status"]) == [[""]] diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 1e0c3b550..1c76eb219 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -30,7 +30,7 @@ def test_order_refresh(): with taddons.context() as tctx: tctx.configure(v, view_order="time") v.add([tf]) - tf.request.timestamp_start = 1 + tf.request.timestamp_start = 10 assert not sargs v.update([tf]) assert sargs @@ -41,7 +41,7 @@ def test_order_generators(): tf = tflow.tflow(resp=True) rs = view.OrderRequestStart(v) - assert rs.generate(tf) == 0 + assert rs.generate(tf) == 1 rm = view.OrderRequestMethod(v) assert rm.generate(tf) == tf.request.method diff --git a/test/mitmproxy/net/http/test_response.py b/test/mitmproxy/net/http/test_response.py index fa1770feb..a77435c93 100644 --- a/test/mitmproxy/net/http/test_response.py +++ b/test/mitmproxy/net/http/test_response.py @@ -150,10 +150,10 @@ class TestResponseUtils: n = time.time() r.headers["date"] = email.utils.formatdate(n) pre = r.headers["date"] - r.refresh(n) + r.refresh(1) assert pre == r.headers["date"] - r.refresh(n + 60) + r.refresh(61) d = email.utils.parsedate_tz(r.headers["date"]) d = email.utils.mktime_tz(d) # Weird that this is not exact... diff --git a/web/src/js/__tests__/ducks/_tflow.js b/web/src/js/__tests__/ducks/_tflow.js index 44b323427..8599200de 100644 --- a/web/src/js/__tests__/ducks/_tflow.js +++ b/web/src/js/__tests__/ducks/_tflow.js @@ -45,8 +45,8 @@ export default function(){ "port": 22, "pretty_host": "address", "scheme": "http", - "timestamp_end": null, - "timestamp_start": null + "timestamp_end": 2, + "timestamp_start": 1 }, "response": { "contentHash": "ab530a13e45914982b79f9b7e3fba994cfd1f3fb22f71cea1afbf02b460c6d1d",