From 3976ea01582717b8bf0e12cfe1160e7e1d8f0121 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Thu, 10 Aug 2017 14:37:19 +0800 Subject: [PATCH 01/13] [web] Import static.js in index.html This is necessary after we introducing the static mode, or it will raise an "undefined" exception. --- mitmproxy/tools/web/templates/index.html | 1 + 1 file changed, 1 insertion(+) diff --git a/mitmproxy/tools/web/templates/index.html b/mitmproxy/tools/web/templates/index.html index db9d2ecb7..d2d017764 100644 --- a/mitmproxy/tools/web/templates/index.html +++ b/mitmproxy/tools/web/templates/index.html @@ -7,6 +7,7 @@ + From bf0834051da57da604db2c4068665243c9871bc7 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Thu, 10 Aug 2017 14:39:19 +0800 Subject: [PATCH 02/13] [web] Attempt of static viewer addon. --- mitmproxy/tools/web/static_viewer.py | 52 ++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 mitmproxy/tools/web/static_viewer.py diff --git a/mitmproxy/tools/web/static_viewer.py b/mitmproxy/tools/web/static_viewer.py new file mode 100644 index 000000000..ddee6c23a --- /dev/null +++ b/mitmproxy/tools/web/static_viewer.py @@ -0,0 +1,52 @@ +import os.path +import shutil + +from mitmproxy import ctx +from mitmproxy import flow + + +class StaticViewer: + def __init__(self): + self.active_flows = set() # type: Set[flow.Flow] + + def save(self, path: str) -> None: + """ + Save the files for the static web view. + """ + static_path = os.path.join(os.path.dirname(__file__), 'static') + index_path = os.path.join(os.path.dirname(__file__), 'templates', 'index.html') + path = os.path.expanduser(path) + # We want to overwrite the static files to keep track of the update. + try: + shutil.copytree(static_path, os.path.join(path, 'static'), + ignore=shutil.ignore_patterns('static.js')) + except FileExistsError: + shutil.rmtree(os.path.join(path, 'static')) + shutil.copytree(static_path, os.path.join(path, 'static'), + ignore=shutil.ignore_patterns('static.js')) + + index_template = open(index_path, 'r') + index = open(os.path.join(path, 'index.html'), 'w') + # Change the resource files to relative path. + index.write(index_template.read().replace('/static/', './static/')) + index_template.close() + index.close() + + static_template = open(os.path.join(static_path, 'static.js'), 'r') + static = open(os.path.join(path, 'static', 'static.js'), 'w') + # Turn on MITMWEB_STATIC variable + static.write(static_template.read().replace('false', 'true')) + static_template.close() + static.close() + + def load(self, loader): + loader.add_option( + "web_static_viewer", str, "", + "The path to output a static viewer." + ) + + def configure(self, updated): + if "web_static_viewer" in updated and ctx.options.web_static_viewer: + self.save(ctx.options.web_static_viewer) + + From 1b129b50990ce253f2b9e96902ef2330e804162e Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Sat, 12 Aug 2017 19:25:56 +0800 Subject: [PATCH 03/13] [web] Update static_viewer addon. When the addon is triggered, we first copy the static resource files to the output dir, then load the `flows` file and dump json files. --- mitmproxy/tools/web/static_viewer.py | 120 +++++++++++++++++++-------- 1 file changed, 86 insertions(+), 34 deletions(-) diff --git a/mitmproxy/tools/web/static_viewer.py b/mitmproxy/tools/web/static_viewer.py index ddee6c23a..20815fcd5 100644 --- a/mitmproxy/tools/web/static_viewer.py +++ b/mitmproxy/tools/web/static_viewer.py @@ -1,52 +1,104 @@ import os.path import shutil +import json +from typing import Optional +from mitmproxy import io from mitmproxy import ctx -from mitmproxy import flow +from mitmproxy import flowfilter +from mitmproxy import contentviews +from mitmproxy.tools.web.app import flow_to_json class StaticViewer: def __init__(self): - self.active_flows = set() # type: Set[flow.Flow] - - def save(self, path: str) -> None: - """ - Save the files for the static web view. - """ - static_path = os.path.join(os.path.dirname(__file__), 'static') - index_path = os.path.join(os.path.dirname(__file__), 'templates', 'index.html') - path = os.path.expanduser(path) - # We want to overwrite the static files to keep track of the update. - try: - shutil.copytree(static_path, os.path.join(path, 'static'), - ignore=shutil.ignore_patterns('static.js')) - except FileExistsError: - shutil.rmtree(os.path.join(path, 'static')) - shutil.copytree(static_path, os.path.join(path, 'static'), - ignore=shutil.ignore_patterns('static.js')) - - index_template = open(index_path, 'r') - index = open(os.path.join(path, 'index.html'), 'w') - # Change the resource files to relative path. - index.write(index_template.read().replace('/static/', './static/')) - index_template.close() - index.close() - - static_template = open(os.path.join(static_path, 'static.js'), 'r') - static = open(os.path.join(path, 'static', 'static.js'), 'w') - # Turn on MITMWEB_STATIC variable - static.write(static_template.read().replace('false', 'true')) - static_template.close() - static.close() + self.flows = set() # type: Set[flow.Flow] + self.path = '' + self.flows_path = '' def load(self, loader): loader.add_option( - "web_static_viewer", str, "", + "web_static_viewer", Optional[str], "", "The path to output a static viewer." ) def configure(self, updated): if "web_static_viewer" in updated and ctx.options.web_static_viewer: - self.save(ctx.options.web_static_viewer) + self.path = os.path.expanduser(ctx.options.web_static_viewer) + if "rfile" in updated and ctx.options.rfile: + self.flows_path = os.path.expanduser(ctx.options.rfile) + if self.flows_path and self.path: + self.save_static() + self.load_flows() + self.save_flows() + self.save_filter_help() + self.save_flows_content() + def load_flows(self) -> None: + with open(self.flows_path, 'rb') as file: + for i in io.FlowReader(file).stream(): + self.flows.add(i) + + def save_flows(self) -> None: + with open(os.path.join(self.path, '_flows'), 'w') as file: + flows = [] + for f in self.flows: + flows.append(flow_to_json(f)) + json.dump(flows, file) + + def save_flows_content(self) -> None: + for f in self.flows: + for m in ('request', 'response'): + message = getattr(f, m) + path = os.path.join(self.path, 'flows', f.id, m) + if not os.path.exists(path): + os.makedirs(path) + with open(os.path.join(path, '_content'), 'wb') as file: + file.write(message.raw_content) + + # content_view + view_path = os.path.join(path, 'content') + if not os.path.exists(view_path): + os.makedirs(view_path) + description, lines, error = contentviews.get_message_content_view( + 'Auto', message + ) + with open(os.path.join(view_path, 'Auto'), 'w') as file: + json.dump(dict( + lines=list(lines), + description=description + ), file) + + def save_static(self) -> None: + """ + Save the files for the static web view. + """ + static_path = os.path.join(os.path.dirname(__file__), 'static') + index_path = os.path.join(os.path.dirname(__file__), 'templates', 'index.html') + # We want to overwrite the static files to keep track of the update. + try: + shutil.copytree(static_path, os.path.join(self.path, 'static'), + ignore=shutil.ignore_patterns('static.js')) + except FileExistsError: + shutil.rmtree(os.path.join(self.path, 'static')) + shutil.copytree(static_path, os.path.join(self.path, 'static'), + ignore=shutil.ignore_patterns('static.js')) + + index_template = open(index_path, 'r') + index = open(os.path.join(self.path, 'index.html'), 'w') + # Change the resource files to relative path. + index.write(index_template.read()) + index_template.close() + index.close() + + static_template = open(os.path.join(static_path, 'static.js'), 'r') + static = open(os.path.join(self.path, 'static', 'static.js'), 'w') + # Turn on MITMWEB_STATIC variable + static.write(static_template.read().replace('false', 'true')) + static_template.close() + static.close() + + def save_filter_help(self) -> None: + with open(os.path.join(self.path, '_filter-help'), 'w') as file: + json.dump(dict(commands=flowfilter.help), file) From e925015d10e554534b64f8b9fc16588a4c90fde1 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Sat, 12 Aug 2017 19:36:43 +0800 Subject: [PATCH 04/13] [web] Update fetch api to suite the static mode. --- web/src/js/flow/utils.js | 9 ++++++++- web/src/js/utils.js | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/web/src/js/flow/utils.js b/web/src/js/flow/utils.js index 3c38058ee..9d4e10124 100644 --- a/web/src/js/flow/utils.js +++ b/web/src/js/flow/utils.js @@ -49,7 +49,14 @@ export var MessageUtils = { } else if (message === flow.response) { message = "response"; } - return `/flows/${flow.id}/${message}/content` + (view ? `/${view}` : ''); + if (global.MITMWEB_STATIC) { + let url = view ? + `/flows/${flow.id}/${message}/content/${view}` : + `/flows/${flow.id}/${message}/_content` + return url; + } else { + return `/flows/${flow.id}/${message}/content` + (view ? `/${view}` : ''); + } } }; diff --git a/web/src/js/utils.js b/web/src/js/utils.js index e8470cec4..68daa1add 100644 --- a/web/src/js/utils.js +++ b/web/src/js/utils.js @@ -81,7 +81,21 @@ function getCookie(name) { } const xsrf = `_xsrf=${getCookie("_xsrf")}`; + export function fetchApi(url, options={}) { + if (global.MITMWEB_STATIC) { + let path = url.split('/'), + filename = path.pop() + filename = '_' + filename + path.push(filename) + let new_url = path.join('/') + return _fetchApi(new_url, options) + } else { + return _fetchApi(url, options) + } +} + +function _fetchApi(url, options={}) { if (options.method && options.method !== "GET") { if (url.indexOf("?") === -1) { url += "?" + xsrf; @@ -96,7 +110,7 @@ export function fetchApi(url, options={}) { }); } -fetchApi.put = (url, json, options) => fetchApi( +fetchApi.put = (url, json, options) => _fetchApi( url, { method: "PUT", From 7d92cdf3bb349b8621d4f96cca87eb2b132b24b4 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Sat, 12 Aug 2017 19:38:52 +0800 Subject: [PATCH 05/13] [web] Hide ContentViewOptions in static mode --- web/src/js/components/FlowView/Messages.jsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web/src/js/components/FlowView/Messages.jsx b/web/src/js/components/FlowView/Messages.jsx index 4a31faf4d..b69dfb69d 100644 --- a/web/src/js/components/FlowView/Messages.jsx +++ b/web/src/js/components/FlowView/Messages.jsx @@ -9,6 +9,7 @@ import ContentView from '../ContentView' import ContentViewOptions from '../ContentView/ContentViewOptions' import ValidateEditor from '../ValueEditor/ValidateEditor' import ValueEditor from '../ValueEditor/ValueEditor' +import HideInStatic from '../common/HideInStatic' import Headers from './Headers' import { startEdit, updateEdit } from '../../ducks/ui/flow' @@ -105,6 +106,7 @@ export class Request extends Component { onContentChange={content => updateFlow({ request: {content}})} message={flow.request}/> + {!noContent &&
uploadContent(flow, content, "request")}/>
} + ) } @@ -172,6 +175,7 @@ export class Response extends Component { message={flow.response} /> + {!noContent &&
} + ) } From cbdddefcc8f3ad19e50da5a31532c8abb0cb36f8 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Mon, 14 Aug 2017 10:11:48 +0800 Subject: [PATCH 06/13] [web] change the path name like `_flows` to `flows.json`. --- mitmproxy/tools/web/static_viewer.py | 12 ++++++------ web/src/js/flow/utils.js | 4 ++-- web/src/js/utils.js | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mitmproxy/tools/web/static_viewer.py b/mitmproxy/tools/web/static_viewer.py index 20815fcd5..23228b3b4 100644 --- a/mitmproxy/tools/web/static_viewer.py +++ b/mitmproxy/tools/web/static_viewer.py @@ -41,7 +41,7 @@ class StaticViewer: self.flows.add(i) def save_flows(self) -> None: - with open(os.path.join(self.path, '_flows'), 'w') as file: + with open(os.path.join(self.path, 'flows.json'), 'w') as file: flows = [] for f in self.flows: flows.append(flow_to_json(f)) @@ -54,8 +54,8 @@ class StaticViewer: path = os.path.join(self.path, 'flows', f.id, m) if not os.path.exists(path): os.makedirs(path) - with open(os.path.join(path, '_content'), 'wb') as file: - file.write(message.raw_content) + with open(os.path.join(path, 'content.json'), 'wb') as content_file: + content_file.write(message.raw_content) # content_view view_path = os.path.join(path, 'content') @@ -64,11 +64,11 @@ class StaticViewer: description, lines, error = contentviews.get_message_content_view( 'Auto', message ) - with open(os.path.join(view_path, 'Auto'), 'w') as file: + with open(os.path.join(view_path, 'Auto.json'), 'w') as view_file: json.dump(dict( lines=list(lines), description=description - ), file) + ), view_file) def save_static(self) -> None: """ @@ -100,5 +100,5 @@ class StaticViewer: static.close() def save_filter_help(self) -> None: - with open(os.path.join(self.path, '_filter-help'), 'w') as file: + with open(os.path.join(self.path, 'filter-help.json'), 'w') as file: json.dump(dict(commands=flowfilter.help), file) diff --git a/web/src/js/flow/utils.js b/web/src/js/flow/utils.js index 9d4e10124..cb37822a4 100644 --- a/web/src/js/flow/utils.js +++ b/web/src/js/flow/utils.js @@ -51,8 +51,8 @@ export var MessageUtils = { } if (global.MITMWEB_STATIC) { let url = view ? - `/flows/${flow.id}/${message}/content/${view}` : - `/flows/${flow.id}/${message}/_content` + `/flows/${flow.id}/${message}/content/${view}.json` : + `/flows/${flow.id}/${message}/content.json` return url; } else { return `/flows/${flow.id}/${message}/content` + (view ? `/${view}` : ''); diff --git a/web/src/js/utils.js b/web/src/js/utils.js index 68daa1add..505f74df9 100644 --- a/web/src/js/utils.js +++ b/web/src/js/utils.js @@ -86,7 +86,7 @@ export function fetchApi(url, options={}) { if (global.MITMWEB_STATIC) { let path = url.split('/'), filename = path.pop() - filename = '_' + filename + filename += '.json' path.push(filename) let new_url = path.join('/') return _fetchApi(new_url, options) From 6560b0dcdba72085ce6c53d68a9e01d293936893 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Tue, 15 Aug 2017 22:21:08 +0800 Subject: [PATCH 07/13] [web] Change the api routing and minor fix. --- mitmproxy/tools/web/app.py | 14 +++++++------- mitmproxy/tools/web/static_viewer.py | 2 +- web/src/js/backends/static.js | 4 +--- web/src/js/flow/utils.js | 9 +-------- web/src/js/utils.js | 18 +++--------------- 5 files changed, 13 insertions(+), 34 deletions(-) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 2a6f6c9e1..52bac8b0f 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -466,10 +466,10 @@ class Application(tornado.web.Application): self.master = master handlers = [ (r"/", IndexHandler), - (r"/filter-help", FilterHelp), + (r"/filter-help(?:\.json)?", FilterHelp), (r"/updates", ClientConnection), - (r"/events", Events), - (r"/flows", Flows), + (r"/events(?:\.json)?", Events), + (r"/flows(?:\.json)?", Flows), (r"/flows/dump", DumpFlows), (r"/flows/resume", ResumeFlows), (r"/flows/kill", KillFlows), @@ -479,13 +479,13 @@ class Application(tornado.web.Application): (r"/flows/(?P[0-9a-f\-]+)/duplicate", DuplicateFlow), (r"/flows/(?P[0-9a-f\-]+)/replay", ReplayFlow), (r"/flows/(?P[0-9a-f\-]+)/revert", RevertFlow), - (r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/content", FlowContent), + (r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/_content", FlowContent), ( - r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/content/(?P[0-9a-zA-Z\-\_]+)", + r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/content/(?P[0-9a-zA-Z\-\_]+)(?:\.json)?", FlowContentView), - (r"/settings", Settings), + (r"/settings(?:\.json)?", Settings), (r"/clear", ClearAll), - (r"/options", Options), + (r"/options(?:\.json)?", Options), (r"/options/save", SaveOptions) ] settings = dict( diff --git a/mitmproxy/tools/web/static_viewer.py b/mitmproxy/tools/web/static_viewer.py index 23228b3b4..01ecc3bbc 100644 --- a/mitmproxy/tools/web/static_viewer.py +++ b/mitmproxy/tools/web/static_viewer.py @@ -54,7 +54,7 @@ class StaticViewer: path = os.path.join(self.path, 'flows', f.id, m) if not os.path.exists(path): os.makedirs(path) - with open(os.path.join(path, 'content.json'), 'wb') as content_file: + with open(os.path.join(path, '_content'), 'wb') as content_file: content_file.write(message.raw_content) # content_view diff --git a/web/src/js/backends/static.js b/web/src/js/backends/static.js index 6657fecf1..7e87a5eda 100644 --- a/web/src/js/backends/static.js +++ b/web/src/js/backends/static.js @@ -11,10 +11,8 @@ export default class StaticBackend { } onOpen() { - this.fetchData("settings") this.fetchData("flows") - this.fetchData("events") - this.fetchData("options") + // this.fetchData("events") # TODO: Add events log to static viewer. } fetchData(resource) { diff --git a/web/src/js/flow/utils.js b/web/src/js/flow/utils.js index cb37822a4..9915a639a 100644 --- a/web/src/js/flow/utils.js +++ b/web/src/js/flow/utils.js @@ -49,14 +49,7 @@ export var MessageUtils = { } else if (message === flow.response) { message = "response"; } - if (global.MITMWEB_STATIC) { - let url = view ? - `/flows/${flow.id}/${message}/content/${view}.json` : - `/flows/${flow.id}/${message}/content.json` - return url; - } else { - return `/flows/${flow.id}/${message}/content` + (view ? `/${view}` : ''); - } + return `/flows/${flow.id}/${message}/` + (view ? `content/${view}.json` : '_content'); } }; diff --git a/web/src/js/utils.js b/web/src/js/utils.js index 505f74df9..3aeba1b19 100644 --- a/web/src/js/utils.js +++ b/web/src/js/utils.js @@ -81,27 +81,15 @@ function getCookie(name) { } const xsrf = `_xsrf=${getCookie("_xsrf")}`; - export function fetchApi(url, options={}) { - if (global.MITMWEB_STATIC) { - let path = url.split('/'), - filename = path.pop() - filename += '.json' - path.push(filename) - let new_url = path.join('/') - return _fetchApi(new_url, options) - } else { - return _fetchApi(url, options) - } -} - -function _fetchApi(url, options={}) { if (options.method && options.method !== "GET") { if (url.indexOf("?") === -1) { url += "?" + xsrf; } else { url += "&" + xsrf; } + } else { + url += '.json' } return fetch(url, { @@ -110,7 +98,7 @@ function _fetchApi(url, options={}) { }); } -fetchApi.put = (url, json, options) => _fetchApi( +fetchApi.put = (url, json, options) => fetchApi( url, { method: "PUT", From 3f497640ab0a69838fb605ed9a4b1cee76e462aa Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Tue, 15 Aug 2017 22:23:21 +0800 Subject: [PATCH 08/13] [web] Update tests. --- test/mitmproxy/tools/web/test_app.py | 10 +++++----- .../__snapshots__/ContentViewOptionsSpec.js.snap | 2 +- .../ContentView/__snapshots__/ContentViewSpec.js.snap | 2 +- .../__snapshots__/DownloadContentButtonSpec.js.snap | 2 +- .../ContentView/__snapshots__/MetaViewsSpec.js.snap | 2 +- .../FlowView/__snapshots__/MessagesSpec.js.snap | 4 ++-- .../components/__snapshots__/ContentViewSpec.js.snap | 2 +- web/src/js/__tests__/flow/utilsSpec.js | 6 +++--- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py index 091ef5e89..aaf949a86 100644 --- a/test/mitmproxy/tools/web/test_app.py +++ b/test/mitmproxy/tools/web/test_app.py @@ -186,7 +186,7 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): f.response.headers["Content-Encoding"] = "ran\x00dom" f.response.headers["Content-Disposition"] = 'inline; filename="filename.jpg"' - r = self.fetch("/flows/42/response/content") + r = self.fetch("/flows/42/response/_content") assert r.body == b"message" assert r.headers["Content-Encoding"] == "random" assert r.headers["Content-Disposition"] == 'attachment; filename="filename.jpg"' @@ -194,17 +194,17 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): del f.response.headers["Content-Disposition"] f.request.path = "/foo/bar.jpg" assert self.fetch( - "/flows/42/response/content" + "/flows/42/response/_content" ).headers["Content-Disposition"] == 'attachment; filename=bar.jpg' f.response.content = b"" - assert self.fetch("/flows/42/response/content").code == 400 + assert self.fetch("/flows/42/response/_content").code == 400 f.revert() def test_update_flow_content(self): assert self.fetch( - "/flows/42/request/content", + "/flows/42/request/_content", method="POST", body="new" ).code == 200 @@ -222,7 +222,7 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): b'--somefancyboundary--\r\n' ) assert self.fetch( - "/flows/42/request/content", + "/flows/42/request/_content", method="POST", headers={"Content-Type": 'multipart/form-data; boundary="somefancyboundary"'}, body=body diff --git a/web/src/js/__tests__/components/ContentView/__snapshots__/ContentViewOptionsSpec.js.snap b/web/src/js/__tests__/components/ContentView/__snapshots__/ContentViewOptionsSpec.js.snap index e3561ec1a..01fab0a7e 100644 --- a/web/src/js/__tests__/components/ContentView/__snapshots__/ContentViewOptionsSpec.js.snap +++ b/web/src/js/__tests__/components/ContentView/__snapshots__/ContentViewOptionsSpec.js.snap @@ -13,7 +13,7 @@ exports[`ContentViewOptions Component should render correctly 1`] = `   `; diff --git a/web/src/js/__tests__/components/ContentView/__snapshots__/DownloadContentButtonSpec.js.snap b/web/src/js/__tests__/components/ContentView/__snapshots__/DownloadContentButtonSpec.js.snap index 66900ca4e..4c578a0c2 100644 --- a/web/src/js/__tests__/components/ContentView/__snapshots__/DownloadContentButtonSpec.js.snap +++ b/web/src/js/__tests__/components/ContentView/__snapshots__/DownloadContentButtonSpec.js.snap @@ -3,7 +3,7 @@ exports[`DownloadContentButton Component should render correctly 1`] = ` { let msg = "foo", view = "bar", flow = { request: msg, id: 1} expect(utils.MessageUtils.getContentURL(flow, msg, view)).toEqual( - "/flows/1/request/content/bar" + "/flows/1/request/content/bar.json" ) expect(utils.MessageUtils.getContentURL(flow, msg, '')).toEqual( - "/flows/1/request/content" + "/flows/1/request/_content" ) // response flow = {response: msg, id: 2} expect(utils.MessageUtils.getContentURL(flow, msg, view)).toEqual( - "/flows/2/response/content/bar" + "/flows/2/response/content/bar.json" ) }) }) From 4fb255a7afda3c2bf32e6d6d305aee1cfc999ba6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 16 Aug 2017 19:09:02 +0200 Subject: [PATCH 09/13] make static viewer more testable, print slow contentviews --- mitmproxy/addons/__init__.py | 2 + mitmproxy/addons/static_viewer.py | 93 ++++++++++++++++++++++++ mitmproxy/tools/web/app.py | 3 +- mitmproxy/tools/web/static_viewer.py | 104 --------------------------- 4 files changed, 96 insertions(+), 106 deletions(-) create mode 100644 mitmproxy/addons/static_viewer.py delete mode 100644 mitmproxy/tools/web/static_viewer.py diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index 62135765c..e46c215be 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -14,6 +14,7 @@ from mitmproxy.addons import replace from mitmproxy.addons import script from mitmproxy.addons import serverplayback from mitmproxy.addons import setheaders +from mitmproxy.addons import static_viewer from mitmproxy.addons import stickyauth from mitmproxy.addons import stickycookie from mitmproxy.addons import streambodies @@ -39,6 +40,7 @@ def default_addons(): script.ScriptLoader(), serverplayback.ServerPlayback(), setheaders.SetHeaders(), + static_viewer.StaticViewer(), stickyauth.StickyAuth(), stickycookie.StickyCookie(), streambodies.StreamBodies(), diff --git a/mitmproxy/addons/static_viewer.py b/mitmproxy/addons/static_viewer.py new file mode 100644 index 000000000..fddb36a42 --- /dev/null +++ b/mitmproxy/addons/static_viewer.py @@ -0,0 +1,93 @@ +import json +import os.path +import pathlib +import shutil +import time +import typing + +from mitmproxy import contentviews +from mitmproxy import ctx +from mitmproxy import flowfilter +from mitmproxy import io, flow +from mitmproxy.tools.web.app import flow_to_json + +web_dir = pathlib.Path(__file__).absolute().parent.parent / "tools" / "web" + + +def save_static(path: pathlib.Path) -> None: + """ + Save the files for the static web view. + """ + # We want to overwrite the static files to keep track of the update. + if (path / "static").exists(): + shutil.rmtree(str(path / "static")) + shutil.copytree(str(web_dir / "static"), str(path / "static")) + shutil.copyfile(str(web_dir / 'templates' / 'index.html'), str(path / "index.html")) + + with open(web_dir / "static" / "static.js", "w") as f: + f.write("MITMWEB_STATIC = true;") + + +def save_filter_help(path: pathlib.Path) -> None: + with open(path / 'filter-help.json', 'w') as f: + json.dump(dict(commands=flowfilter.help), f) + + +def save_flows(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: + with open(path / 'flows.json', 'w') as f: + json.dump( + [flow_to_json(f) for f in flows], + f + ) + + +def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: + for f in flows: + for m in ('request', 'response'): + message = getattr(f, m) + message_path = path / "flows" / f.id / m + os.makedirs(str(message_path / "content"), exist_ok=True) + + with open(message_path / '_content', 'wb') as f: + # don't use raw_content here as this is served with a default content type + f.write(message.content) + + # content_view + t = time.time() + description, lines, error = contentviews.get_message_content_view( + 'Auto', message + ) + if time.time() - t > 0.1: + ctx.log( + "Slow content view: {} took {}s".format( + description.strip(), + round(time.time() - t, 1) + ), + "info" + ) + with open(message_path / "content" / "Auto.json", "w") as f: + json.dump( + dict(lines=list(lines), description=description), + f + ) + + +class StaticViewer: + # TODO: make this a command at some point. + def load(self, loader): + loader.add_option( + "web_static_viewer", typing.Optional[str], "", + "The path to output a static viewer." + ) + + def configure(self, updated): + if "web_static_viewer" in updated and ctx.options.web_static_viewer: + flows = io.read_flows_from_paths([ctx.options.rfile]) + p = pathlib.Path(ctx.options.web_static_viewer).expanduser() + self.export(p, flows) + + def export(self, path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: + save_static(path) + save_filter_help(path) + save_flows(path, flows) + save_flows_content(path, flows) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 52bac8b0f..9c6f7583a 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -5,7 +5,6 @@ import os.path import re from io import BytesIO -import mitmproxy.addons.view import mitmproxy.flow import tornado.escape import tornado.web @@ -149,7 +148,7 @@ class RequestHandler(tornado.web.RequestHandler): return self.request.body @property - def view(self) -> mitmproxy.addons.view.View: + def view(self) -> "mitmproxy.addons.view.View": return self.application.master.view @property diff --git a/mitmproxy/tools/web/static_viewer.py b/mitmproxy/tools/web/static_viewer.py deleted file mode 100644 index 01ecc3bbc..000000000 --- a/mitmproxy/tools/web/static_viewer.py +++ /dev/null @@ -1,104 +0,0 @@ -import os.path -import shutil -import json -from typing import Optional - -from mitmproxy import io -from mitmproxy import ctx -from mitmproxy import flowfilter -from mitmproxy import contentviews -from mitmproxy.tools.web.app import flow_to_json - - -class StaticViewer: - def __init__(self): - self.flows = set() # type: Set[flow.Flow] - self.path = '' - self.flows_path = '' - - def load(self, loader): - loader.add_option( - "web_static_viewer", Optional[str], "", - "The path to output a static viewer." - ) - - def configure(self, updated): - if "web_static_viewer" in updated and ctx.options.web_static_viewer: - self.path = os.path.expanduser(ctx.options.web_static_viewer) - if "rfile" in updated and ctx.options.rfile: - self.flows_path = os.path.expanduser(ctx.options.rfile) - - if self.flows_path and self.path: - self.save_static() - self.load_flows() - self.save_flows() - self.save_filter_help() - self.save_flows_content() - - def load_flows(self) -> None: - with open(self.flows_path, 'rb') as file: - for i in io.FlowReader(file).stream(): - self.flows.add(i) - - def save_flows(self) -> None: - with open(os.path.join(self.path, 'flows.json'), 'w') as file: - flows = [] - for f in self.flows: - flows.append(flow_to_json(f)) - json.dump(flows, file) - - def save_flows_content(self) -> None: - for f in self.flows: - for m in ('request', 'response'): - message = getattr(f, m) - path = os.path.join(self.path, 'flows', f.id, m) - if not os.path.exists(path): - os.makedirs(path) - with open(os.path.join(path, '_content'), 'wb') as content_file: - content_file.write(message.raw_content) - - # content_view - view_path = os.path.join(path, 'content') - if not os.path.exists(view_path): - os.makedirs(view_path) - description, lines, error = contentviews.get_message_content_view( - 'Auto', message - ) - with open(os.path.join(view_path, 'Auto.json'), 'w') as view_file: - json.dump(dict( - lines=list(lines), - description=description - ), view_file) - - def save_static(self) -> None: - """ - Save the files for the static web view. - """ - static_path = os.path.join(os.path.dirname(__file__), 'static') - index_path = os.path.join(os.path.dirname(__file__), 'templates', 'index.html') - # We want to overwrite the static files to keep track of the update. - try: - shutil.copytree(static_path, os.path.join(self.path, 'static'), - ignore=shutil.ignore_patterns('static.js')) - except FileExistsError: - shutil.rmtree(os.path.join(self.path, 'static')) - shutil.copytree(static_path, os.path.join(self.path, 'static'), - ignore=shutil.ignore_patterns('static.js')) - - index_template = open(index_path, 'r') - index = open(os.path.join(self.path, 'index.html'), 'w') - # Change the resource files to relative path. - index.write(index_template.read()) - index_template.close() - index.close() - - static_template = open(os.path.join(static_path, 'static.js'), 'r') - static = open(os.path.join(self.path, 'static', 'static.js'), 'w') - # Turn on MITMWEB_STATIC variable - static.write(static_template.read().replace('false', 'true')) - static_template.close() - static.close() - - def save_filter_help(self) -> None: - with open(os.path.join(self.path, 'filter-help.json'), 'w') as file: - json.dump(dict(commands=flowfilter.help), file) From d687ebc1ead4dc87237598e0ce1ac8fdcbaaacd1 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Thu, 17 Aug 2017 09:01:40 +0800 Subject: [PATCH 10/13] [web] Minor fixes. --- mitmproxy/addons/static_viewer.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/mitmproxy/addons/static_viewer.py b/mitmproxy/addons/static_viewer.py index fddb36a42..cf1ae9aeb 100644 --- a/mitmproxy/addons/static_viewer.py +++ b/mitmproxy/addons/static_viewer.py @@ -24,17 +24,17 @@ def save_static(path: pathlib.Path) -> None: shutil.copytree(str(web_dir / "static"), str(path / "static")) shutil.copyfile(str(web_dir / 'templates' / 'index.html'), str(path / "index.html")) - with open(web_dir / "static" / "static.js", "w") as f: + with open(str(path / "static" / "static.js"), "w") as f: f.write("MITMWEB_STATIC = true;") def save_filter_help(path: pathlib.Path) -> None: - with open(path / 'filter-help.json', 'w') as f: + with open(str(path / 'filter-help.json'), 'w') as f: json.dump(dict(commands=flowfilter.help), f) def save_flows(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: - with open(path / 'flows.json', 'w') as f: + with open(str(path / 'flows.json'), 'w') as f: json.dump( [flow_to_json(f) for f in flows], f @@ -42,14 +42,17 @@ def save_flows(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: - for f in flows: + for flow in flows: for m in ('request', 'response'): - message = getattr(f, m) - message_path = path / "flows" / f.id / m + message = getattr(flow, m) + message_path = path / "flows" / flow.id / m os.makedirs(str(message_path / "content"), exist_ok=True) - with open(message_path / '_content', 'wb') as f: + with open(str(message_path / '_content'), 'wb') as f: # don't use raw_content here as this is served with a default content type + if not message: + # skip missing message + continue f.write(message.content) # content_view @@ -65,7 +68,7 @@ def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> ), "info" ) - with open(message_path / "content" / "Auto.json", "w") as f: + with open(str(message_path / "content" / "Auto.json"), "w") as f: json.dump( dict(lines=list(lines), description=description), f From af30930ae8ea10b1be3d062cc12481d37c377aa6 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Thu, 17 Aug 2017 10:53:09 +0800 Subject: [PATCH 11/13] [web] Add tests for StaticViewer and minor fixes. --- mitmproxy/addons/static_viewer.py | 29 +++++----- test/mitmproxy/addons/test_static_viewer.py | 60 +++++++++++++++++++++ 2 files changed, 76 insertions(+), 13 deletions(-) create mode 100644 test/mitmproxy/addons/test_static_viewer.py diff --git a/mitmproxy/addons/static_viewer.py b/mitmproxy/addons/static_viewer.py index cf1ae9aeb..a46292304 100644 --- a/mitmproxy/addons/static_viewer.py +++ b/mitmproxy/addons/static_viewer.py @@ -42,24 +42,27 @@ def save_flows(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: - for flow in flows: + for f in flows: for m in ('request', 'response'): - message = getattr(flow, m) - message_path = path / "flows" / flow.id / m + message = getattr(f, m) + message_path = path / "flows" / f.id / m os.makedirs(str(message_path / "content"), exist_ok=True) - with open(str(message_path / '_content'), 'wb') as f: + with open(str(message_path / '_content'), 'wb') as content_file: # don't use raw_content here as this is served with a default content type - if not message: - # skip missing message - continue - f.write(message.content) + if message: + content_file.write(message.content) + else: + content_file.write(b'No content.') # content_view t = time.time() - description, lines, error = contentviews.get_message_content_view( - 'Auto', message - ) + if message: + description, lines, error = contentviews.get_message_content_view( + 'Auto', message + ) + else: + description, lines = 'No content.', [] if time.time() - t > 0.1: ctx.log( "Slow content view: {} took {}s".format( @@ -68,10 +71,10 @@ def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> ), "info" ) - with open(str(message_path / "content" / "Auto.json"), "w") as f: + with open(str(message_path / "content" / "Auto.json"), "w") as content_view_file: json.dump( dict(lines=list(lines), description=description), - f + content_view_file ) diff --git a/test/mitmproxy/addons/test_static_viewer.py b/test/mitmproxy/addons/test_static_viewer.py new file mode 100644 index 000000000..7e34a9fc0 --- /dev/null +++ b/test/mitmproxy/addons/test_static_viewer.py @@ -0,0 +1,60 @@ +import json + +from mitmproxy.test import taddons +from mitmproxy.test import tflow + +from mitmproxy import flowfilter +from mitmproxy.tools.web.app import flow_to_json + +from mitmproxy.addons import static_viewer +from mitmproxy.addons import save + + +def test_save_static(tmpdir): + tmpdir.mkdir('static') + static_viewer.save_static(tmpdir) + assert len(tmpdir.listdir()) == 2 + assert tmpdir.join('index.html').check(file=1) + assert tmpdir.join('static/static.js').read() == 'MITMWEB_STATIC = true;' + + +def test_save_filter_help(tmpdir): + static_viewer.save_filter_help(tmpdir) + f = tmpdir.join('/filter-help.json') + assert f.check(file=1) + assert f.read() == json.dumps(dict(commands=flowfilter.help)) + + +def test_save_flows(tmpdir): + flows = [tflow.tflow(req=True, resp=None), tflow.tflow(req=True, resp=True)] + static_viewer.save_flows(tmpdir, flows) + assert tmpdir.join('flows.json').check(file=1) + assert tmpdir.join('flows.json').read() == json.dumps([flow_to_json(f) for f in flows]) + + +def test_save_flows_content(tmpdir): + flows = [tflow.tflow(req=True, resp=None), tflow.tflow(req=True, resp=True)] + static_viewer.save_flows_content(tmpdir, flows) + flows_path = tmpdir.join('flows') + assert len(flows_path.listdir()) == len(flows) + for p in flows_path.listdir(): + assert p.join('request').check(dir=1) + assert p.join('response').check(dir=1) + assert p.join('request/_content').check(file=1) + assert p.join('request/content').check(dir=1) + assert p.join('response/_content').check(file=1) + assert p.join('response/content').check(dir=1) + assert p.join('request/content/Auto.json').check(file=1) + assert p.join('response/content/Auto.json').check(file=1) + + +def test_static_viewer(tmpdir): + s = static_viewer.StaticViewer() + sa = save.Save() + with taddons.context() as tctx: + sa.save([tflow.tflow(resp=True)], str(tmpdir.join('foo'))) + tctx.master.addons.add(s) + tctx.configure(s, web_static_viewer=str(tmpdir), rfile=str(tmpdir.join('foo'))) + assert tmpdir.join('index.html').check(file=1) + assert tmpdir.join('static').check(dir=1) + assert tmpdir.join('flows').check(dir=1) From d008f179c3e6fa620c6babc6cedf8e7df9093827 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Thu, 17 Aug 2017 21:40:52 +0800 Subject: [PATCH 12/13] [web] Static_viewer coverage ++. --- test/mitmproxy/addons/test_static_viewer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/mitmproxy/addons/test_static_viewer.py b/test/mitmproxy/addons/test_static_viewer.py index 7e34a9fc0..bb2b6777e 100644 --- a/test/mitmproxy/addons/test_static_viewer.py +++ b/test/mitmproxy/addons/test_static_viewer.py @@ -1,4 +1,5 @@ import json +from unittest import mock from mitmproxy.test import taddons from mitmproxy.test import tflow @@ -32,9 +33,11 @@ def test_save_flows(tmpdir): assert tmpdir.join('flows.json').read() == json.dumps([flow_to_json(f) for f in flows]) -def test_save_flows_content(tmpdir): +@mock.patch('mitmproxy.ctx.log') +def test_save_flows_content(ctx, tmpdir): flows = [tflow.tflow(req=True, resp=None), tflow.tflow(req=True, resp=True)] - static_viewer.save_flows_content(tmpdir, flows) + with mock.patch('time.time', mock.Mock(side_effect=[1, 2, 2] * 4)): + static_viewer.save_flows_content(tmpdir, flows) flows_path = tmpdir.join('flows') assert len(flows_path.listdir()) == len(flows) for p in flows_path.listdir(): From 2baa2c4049ab6b2801b3836b28b3e9df815a0075 Mon Sep 17 00:00:00 2001 From: Matthew Shao Date: Mon, 21 Aug 2017 09:27:49 +0800 Subject: [PATCH 13/13] [web] Move static viewer from default addon to WebMaster. --- mitmproxy/addons/__init__.py | 2 -- mitmproxy/tools/web/master.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index e46c215be..62135765c 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -14,7 +14,6 @@ from mitmproxy.addons import replace from mitmproxy.addons import script from mitmproxy.addons import serverplayback from mitmproxy.addons import setheaders -from mitmproxy.addons import static_viewer from mitmproxy.addons import stickyauth from mitmproxy.addons import stickycookie from mitmproxy.addons import streambodies @@ -40,7 +39,6 @@ def default_addons(): script.ScriptLoader(), serverplayback.ServerPlayback(), setheaders.SetHeaders(), - static_viewer.StaticViewer(), stickyauth.StickyAuth(), stickycookie.StickyCookie(), streambodies.StreamBodies(), diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index b13aeff9e..c391a1cd4 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -12,6 +12,7 @@ from mitmproxy.addons import readfile from mitmproxy.addons import termlog from mitmproxy.addons import view from mitmproxy.addons import termstatus +from mitmproxy.addons import static_viewer from mitmproxy.options import Options # noqa from mitmproxy.tools.web import app, webaddons @@ -37,6 +38,7 @@ class WebMaster(master.Master): webaddons.WebAddon(), intercept.Intercept(), readfile.ReadFile(), + static_viewer.StaticViewer(), self.view, self.events, )