From dae01ad62329c74fa9373d2052c6bf7f879fa523 Mon Sep 17 00:00:00 2001 From: Michael McKeirnan Date: Thu, 14 Nov 2019 23:52:45 -0800 Subject: [PATCH 1/4] Adding export raw http response Adding a new export type for raw http response, and changing export raw to export raw_request to distinguish between the two. This is a proposed change for https://github.com/mitmproxy/mitmproxy/issues/3701 --- mitmproxy/addons/export.py | 16 ++++++++++-- test/mitmproxy/addons/test_export.py | 38 ++++++++++++++++++++++------ 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 2776118ab..704f68da0 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -23,6 +23,14 @@ def cleanup_request(f: flow.Flow): return request +def cleanup_response(f: flow.Flow): + if not hasattr(f, "response"): + raise exceptions.CommandError("Can't export flow with no response.") + response = f.response.copy() # type: ignore + response.decode(strict=False) + return response + + def curl_command(f: flow.Flow) -> str: data = "curl " request = cleanup_request(f) @@ -53,14 +61,18 @@ def httpie_command(f: flow.Flow) -> str: return data -def raw(f: flow.Flow) -> bytes: +def raw_request(f: flow.Flow) -> bytes: return assemble.assemble_request(cleanup_request(f)) # type: ignore +def raw_response(f: flow.Flow) -> bytes: + return assemble.assemble_response(cleanup_response(f)) # type: ignore + formats = dict( curl = curl_command, httpie = httpie_command, - raw = raw, + raw_request = raw_request, + raw_response = raw_response, ) diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index c86e0c7da..1ea4f2563 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -17,6 +17,12 @@ def get_request(): req=tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz")) +@pytest.fixture +def get_response(): + return tflow.tflow( + resp=tutils.tresp(status_code=404, content=b"Test Response Body")) + + @pytest.fixture def post_request(): return tflow.tflow( @@ -79,13 +85,21 @@ class TestExportHttpieCommand: export.httpie_command(tcp_flow) -class TestRaw: +class TestRawRequest: def test_get(self, get_request): - assert b"header: qvalue" in export.raw(get_request) + assert b"header: qvalue" in export.raw_request(get_request) def test_tcp(self, tcp_flow): with pytest.raises(exceptions.CommandError): - export.raw(tcp_flow) + export.raw_request(tcp_flow) + +class TestRawResponse: + def test_get(self, get_response): + assert b"header-response: svalue" in export.raw_response(get_response) + + def test_tcp(self, tcp_flow): + with pytest.raises(exceptions.CommandError): + export.raw_response(tcp_flow) def qr(f): @@ -97,11 +111,15 @@ def test_export(tmpdir): f = str(tmpdir.join("path")) e = export.Export() with taddons.context(): - assert e.formats() == ["curl", "httpie", "raw"] + assert e.formats() == ["curl", "httpie", "raw_request", "raw_response"] with pytest.raises(exceptions.CommandError): e.file("nonexistent", tflow.tflow(resp=True), f) - e.file("raw", tflow.tflow(resp=True), f) + e.file("raw_request", tflow.tflow(resp=True), f) + assert qr(f) + os.unlink(f) + + e.file("raw_response", tflow.tflow(resp=True), f) assert qr(f) os.unlink(f) @@ -126,7 +144,7 @@ async def test_export_open(exception, log_message, tmpdir): with taddons.context() as tctx: with mock.patch("mitmproxy.addons.export.open") as m: m.side_effect = exception(log_message) - e.file("raw", tflow.tflow(resp=True), f) + e.file("raw_request", tflow.tflow(resp=True), f) assert await tctx.master.await_log(log_message, level="error") @@ -138,7 +156,11 @@ async def test_clip(tmpdir): e.clip("nonexistent", tflow.tflow(resp=True)) with mock.patch('pyperclip.copy') as pc: - e.clip("raw", tflow.tflow(resp=True)) + e.clip("raw_request", tflow.tflow(resp=True)) + assert pc.called + + with mock.patch('pyperclip.copy') as pc: + e.clip("raw_response", tflow.tflow(resp=True)) assert pc.called with mock.patch('pyperclip.copy') as pc: @@ -153,5 +175,5 @@ async def test_clip(tmpdir): log_message = "Pyperclip could not find a " \ "copy/paste mechanism for your system." pc.side_effect = pyperclip.PyperclipException(log_message) - e.clip("raw", tflow.tflow(resp=True)) + e.clip("raw_request", tflow.tflow(resp=True)) assert await tctx.master.await_log(log_message, level="error") From a6e8b930c9aac350cd1701e5e7fe4e7ca7e1ba3c Mon Sep 17 00:00:00 2001 From: Michael McKeirnan Date: Sat, 16 Nov 2019 01:20:50 -0800 Subject: [PATCH 2/4] Adding raw_request and raw_response to export This is a proposed change for https://github.com/mitmproxy/mitmproxy/issues/3701 which alters the behavior of a raw http export to include both the request and the response. Additionally, this introduces two new export options "raw_request" and "raw_response" which allow for exporting the raw HTTP request or response individually. --- mitmproxy/addons/export.py | 26 ++++++++++++--- test/mitmproxy/addons/test_export.py | 48 +++++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 704f68da0..528ccbf6e 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -12,7 +12,7 @@ import pyperclip def cleanup_request(f: flow.Flow): - if not hasattr(f, "request"): + if not hasattr(f, "request") or not f.request: # type: ignore raise exceptions.CommandError("Can't export flow with no request.") request = f.request.copy() # type: ignore request.decode(strict=False) @@ -24,7 +24,7 @@ def cleanup_request(f: flow.Flow): def cleanup_response(f: flow.Flow): - if not hasattr(f, "response"): + if not hasattr(f, "response") or not f.response: # type: ignore raise exceptions.CommandError("Can't export flow with no response.") response = f.response.copy() # type: ignore response.decode(strict=False) @@ -62,15 +62,33 @@ def httpie_command(f: flow.Flow) -> str: def raw_request(f: flow.Flow) -> bytes: - return assemble.assemble_request(cleanup_request(f)) # type: ignore + return assemble.assemble_request(cleanup_request(f)) + def raw_response(f: flow.Flow) -> bytes: - return assemble.assemble_response(cleanup_response(f)) # type: ignore + return assemble.assemble_response(cleanup_response(f)) + + +def raw(f: flow.Flow, separator=b"\r\n\r\n") -> bytes: + """Return either the request or response if only one exists, otherwise return both""" + request_present = hasattr(f, "request") and f.request # type: ignore + response_present = hasattr(f, "response") and f.response # type: ignore + + if not (request_present or response_present): + raise exceptions.CommandError("Can't export flow with no request or response.") + + if request_present and response_present: + return b"".join([raw_request(f), separator, raw_response(f)]) + elif not request_present: + return raw_response(f) + else: + return raw_request(f) formats = dict( curl = curl_command, httpie = httpie_command, + raw = raw, raw_request = raw_request, raw_response = raw_response, ) diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index 1ea4f2563..b0fbeb45e 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -23,6 +23,13 @@ def get_response(): resp=tutils.tresp(status_code=404, content=b"Test Response Body")) +@pytest.fixture +def get_flow(): + return tflow.tflow( + req=tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz"), + resp=tutils.tresp(status_code=404, content=b"Test Response Body")) + + @pytest.fixture def post_request(): return tflow.tflow( @@ -85,18 +92,51 @@ class TestExportHttpieCommand: export.httpie_command(tcp_flow) -class TestRawRequest: - def test_get(self, get_request): - assert b"header: qvalue" in export.raw_request(get_request) +class TestRaw: + def test_req_and_resp_present(self, get_flow): + assert b"header: qvalue" in export.raw(get_flow) + assert b"header-response: svalue" in export.raw(get_flow) + + def test_get_request_present(self, get_request): + assert b"header: qvalue" in export.raw(get_request) + + def test_get_response_present(self, get_response): + delattr(get_response, 'request') + assert b"header-response: svalue" in export.raw(get_response) + + def test_missing_both(self, get_request): + delattr(get_request, 'request') + delattr(get_request, 'response') + with pytest.raises(exceptions.CommandError): + export.raw(get_request) def test_tcp(self, tcp_flow): with pytest.raises(exceptions.CommandError): export.raw_request(tcp_flow) + +class TestRawRequest: + def test_get(self, get_request): + assert b"header: qvalue" in export.raw_request(get_request) + + def test_no_request(self, get_response): + delattr(get_response, 'request') + with pytest.raises(exceptions.CommandError): + export.raw_request(get_response) + + def test_tcp(self, tcp_flow): + with pytest.raises(exceptions.CommandError): + export.raw_request(tcp_flow) + + class TestRawResponse: def test_get(self, get_response): assert b"header-response: svalue" in export.raw_response(get_response) + def test_no_response(self, get_request): + with pytest.raises(exceptions.CommandError): + export.raw_response(get_request) + def test_tcp(self, tcp_flow): with pytest.raises(exceptions.CommandError): export.raw_response(tcp_flow) @@ -111,7 +151,7 @@ def test_export(tmpdir): f = str(tmpdir.join("path")) e = export.Export() with taddons.context(): - assert e.formats() == ["curl", "httpie", "raw_request", "raw_response"] + assert e.formats() == ["curl", "httpie", "raw", "raw_request", "raw_response"] with pytest.raises(exceptions.CommandError): e.file("nonexistent", tflow.tflow(resp=True), f) From bccee15dcbc186d0014809830c5b996a3fb2f442 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 16 Nov 2019 12:09:38 +0100 Subject: [PATCH 3/4] minor type fixes --- mitmproxy/addons/export.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index d87cd7872..7bd0aef0b 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -12,8 +12,8 @@ from mitmproxy.net.http.http1 import assemble from mitmproxy.utils import strutils -def cleanup_request(f: flow.Flow): - if not hasattr(f, "request") or not f.request: # type: ignore +def cleanup_request(f: flow.Flow) -> http.HTTPRequest: + if not hasattr(f, "request") or not f.request: raise exceptions.CommandError("Can't export flow with no request.") assert isinstance(f, http.HTTPFlow) request = f.request.copy() @@ -27,13 +27,15 @@ def cleanup_request(f: flow.Flow): return request -def cleanup_response(f: flow.Flow): - if not hasattr(f, "response") or not f.response: # type: ignore +def cleanup_response(f: flow.Flow)-> http.HTTPResponse: + if not hasattr(f, "response") or not f.response: raise exceptions.CommandError("Can't export flow with no response.") + assert isinstance(f, http.HTTPFlow) response = f.response.copy() # type: ignore response.decode(strict=False) return response + def request_content_for_console(request: http.HTTPRequest) -> str: try: text = request.get_text(strict=True) From ee92477c1bd90cdf1550a537e9a6031997abf62f Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 16 Nov 2019 13:20:17 +0100 Subject: [PATCH 4/4] fix lint --- mitmproxy/addons/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 7bd0aef0b..9d04997ff 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -27,7 +27,7 @@ def cleanup_request(f: flow.Flow) -> http.HTTPRequest: return request -def cleanup_response(f: flow.Flow)-> http.HTTPResponse: +def cleanup_response(f: flow.Flow) -> http.HTTPResponse: if not hasattr(f, "response") or not f.response: raise exceptions.CommandError("Can't export flow with no response.") assert isinstance(f, http.HTTPFlow)