diff --git a/mitmproxy/addons/static_viewer.py b/mitmproxy/addons/static_viewer.py new file mode 100644 index 000000000..a46292304 --- /dev/null +++ b/mitmproxy/addons/static_viewer.py @@ -0,0 +1,99 @@ +import json +import os.path +import pathlib +import shutil +import time +import typing + +from mitmproxy import contentviews +from mitmproxy import ctx +from mitmproxy import flowfilter +from mitmproxy import io, flow +from mitmproxy.tools.web.app import flow_to_json + +web_dir = pathlib.Path(__file__).absolute().parent.parent / "tools" / "web" + + +def save_static(path: pathlib.Path) -> None: + """ + Save the files for the static web view. + """ + # We want to overwrite the static files to keep track of the update. + if (path / "static").exists(): + shutil.rmtree(str(path / "static")) + shutil.copytree(str(web_dir / "static"), str(path / "static")) + shutil.copyfile(str(web_dir / 'templates' / 'index.html'), str(path / "index.html")) + + with open(str(path / "static" / "static.js"), "w") as f: + f.write("MITMWEB_STATIC = true;") + + +def save_filter_help(path: pathlib.Path) -> None: + with open(str(path / 'filter-help.json'), 'w') as f: + json.dump(dict(commands=flowfilter.help), f) + + +def save_flows(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: + with open(str(path / 'flows.json'), 'w') as f: + json.dump( + [flow_to_json(f) for f in flows], + f + ) + + +def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: + for f in flows: + for m in ('request', 'response'): + message = getattr(f, m) + message_path = path / "flows" / f.id / m + os.makedirs(str(message_path / "content"), exist_ok=True) + + with open(str(message_path / '_content'), 'wb') as content_file: + # don't use raw_content here as this is served with a default content type + if message: + content_file.write(message.content) + else: + content_file.write(b'No content.') + + # content_view + t = time.time() + if message: + description, lines, error = contentviews.get_message_content_view( + 'Auto', message + ) + else: + description, lines = 'No content.', [] + if time.time() - t > 0.1: + ctx.log( + "Slow content view: {} took {}s".format( + description.strip(), + round(time.time() - t, 1) + ), + "info" + ) + with open(str(message_path / "content" / "Auto.json"), "w") as content_view_file: + json.dump( + dict(lines=list(lines), description=description), + content_view_file + ) + + +class StaticViewer: + # TODO: make this a command at some point. + def load(self, loader): + loader.add_option( + "web_static_viewer", typing.Optional[str], "", + "The path to output a static viewer." + ) + + def configure(self, updated): + if "web_static_viewer" in updated and ctx.options.web_static_viewer: + flows = io.read_flows_from_paths([ctx.options.rfile]) + p = pathlib.Path(ctx.options.web_static_viewer).expanduser() + self.export(p, flows) + + def export(self, path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: + save_static(path) + save_filter_help(path) + save_flows(path, flows) + save_flows_content(path, flows) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 2a6f6c9e1..9c6f7583a 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -5,7 +5,6 @@ import os.path import re from io import BytesIO -import mitmproxy.addons.view import mitmproxy.flow import tornado.escape import tornado.web @@ -149,7 +148,7 @@ class RequestHandler(tornado.web.RequestHandler): return self.request.body @property - def view(self) -> mitmproxy.addons.view.View: + def view(self) -> "mitmproxy.addons.view.View": return self.application.master.view @property @@ -466,10 +465,10 @@ class Application(tornado.web.Application): self.master = master handlers = [ (r"/", IndexHandler), - (r"/filter-help", FilterHelp), + (r"/filter-help(?:\.json)?", FilterHelp), (r"/updates", ClientConnection), - (r"/events", Events), - (r"/flows", Flows), + (r"/events(?:\.json)?", Events), + (r"/flows(?:\.json)?", Flows), (r"/flows/dump", DumpFlows), (r"/flows/resume", ResumeFlows), (r"/flows/kill", KillFlows), @@ -479,13 +478,13 @@ class Application(tornado.web.Application): (r"/flows/(?P[0-9a-f\-]+)/duplicate", DuplicateFlow), (r"/flows/(?P[0-9a-f\-]+)/replay", ReplayFlow), (r"/flows/(?P[0-9a-f\-]+)/revert", RevertFlow), - (r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/content", FlowContent), + (r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/_content", FlowContent), ( - r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/content/(?P[0-9a-zA-Z\-\_]+)", + r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/content/(?P[0-9a-zA-Z\-\_]+)(?:\.json)?", FlowContentView), - (r"/settings", Settings), + (r"/settings(?:\.json)?", Settings), (r"/clear", ClearAll), - (r"/options", Options), + (r"/options(?:\.json)?", Options), (r"/options/save", SaveOptions) ] settings = dict( diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index b13aeff9e..c391a1cd4 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -12,6 +12,7 @@ from mitmproxy.addons import readfile from mitmproxy.addons import termlog from mitmproxy.addons import view from mitmproxy.addons import termstatus +from mitmproxy.addons import static_viewer from mitmproxy.options import Options # noqa from mitmproxy.tools.web import app, webaddons @@ -37,6 +38,7 @@ class WebMaster(master.Master): webaddons.WebAddon(), intercept.Intercept(), readfile.ReadFile(), + static_viewer.StaticViewer(), self.view, self.events, ) diff --git a/mitmproxy/tools/web/templates/index.html b/mitmproxy/tools/web/templates/index.html index db9d2ecb7..d2d017764 100644 --- a/mitmproxy/tools/web/templates/index.html +++ b/mitmproxy/tools/web/templates/index.html @@ -7,6 +7,7 @@ + diff --git a/test/mitmproxy/addons/test_static_viewer.py b/test/mitmproxy/addons/test_static_viewer.py new file mode 100644 index 000000000..bb2b6777e --- /dev/null +++ b/test/mitmproxy/addons/test_static_viewer.py @@ -0,0 +1,63 @@ +import json +from unittest import mock + +from mitmproxy.test import taddons +from mitmproxy.test import tflow + +from mitmproxy import flowfilter +from mitmproxy.tools.web.app import flow_to_json + +from mitmproxy.addons import static_viewer +from mitmproxy.addons import save + + +def test_save_static(tmpdir): + tmpdir.mkdir('static') + static_viewer.save_static(tmpdir) + assert len(tmpdir.listdir()) == 2 + assert tmpdir.join('index.html').check(file=1) + assert tmpdir.join('static/static.js').read() == 'MITMWEB_STATIC = true;' + + +def test_save_filter_help(tmpdir): + static_viewer.save_filter_help(tmpdir) + f = tmpdir.join('/filter-help.json') + assert f.check(file=1) + assert f.read() == json.dumps(dict(commands=flowfilter.help)) + + +def test_save_flows(tmpdir): + flows = [tflow.tflow(req=True, resp=None), tflow.tflow(req=True, resp=True)] + static_viewer.save_flows(tmpdir, flows) + assert tmpdir.join('flows.json').check(file=1) + assert tmpdir.join('flows.json').read() == json.dumps([flow_to_json(f) for f in flows]) + + +@mock.patch('mitmproxy.ctx.log') +def test_save_flows_content(ctx, tmpdir): + flows = [tflow.tflow(req=True, resp=None), tflow.tflow(req=True, resp=True)] + with mock.patch('time.time', mock.Mock(side_effect=[1, 2, 2] * 4)): + static_viewer.save_flows_content(tmpdir, flows) + flows_path = tmpdir.join('flows') + assert len(flows_path.listdir()) == len(flows) + for p in flows_path.listdir(): + assert p.join('request').check(dir=1) + assert p.join('response').check(dir=1) + assert p.join('request/_content').check(file=1) + assert p.join('request/content').check(dir=1) + assert p.join('response/_content').check(file=1) + assert p.join('response/content').check(dir=1) + assert p.join('request/content/Auto.json').check(file=1) + assert p.join('response/content/Auto.json').check(file=1) + + +def test_static_viewer(tmpdir): + s = static_viewer.StaticViewer() + sa = save.Save() + with taddons.context() as tctx: + sa.save([tflow.tflow(resp=True)], str(tmpdir.join('foo'))) + tctx.master.addons.add(s) + tctx.configure(s, web_static_viewer=str(tmpdir), rfile=str(tmpdir.join('foo'))) + assert tmpdir.join('index.html').check(file=1) + assert tmpdir.join('static').check(dir=1) + assert tmpdir.join('flows').check(dir=1) diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py index 091ef5e89..aaf949a86 100644 --- a/test/mitmproxy/tools/web/test_app.py +++ b/test/mitmproxy/tools/web/test_app.py @@ -186,7 +186,7 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): f.response.headers["Content-Encoding"] = "ran\x00dom" f.response.headers["Content-Disposition"] = 'inline; filename="filename.jpg"' - r = self.fetch("/flows/42/response/content") + r = self.fetch("/flows/42/response/_content") assert r.body == b"message" assert r.headers["Content-Encoding"] == "random" assert r.headers["Content-Disposition"] == 'attachment; filename="filename.jpg"' @@ -194,17 +194,17 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): del f.response.headers["Content-Disposition"] f.request.path = "/foo/bar.jpg" assert self.fetch( - "/flows/42/response/content" + "/flows/42/response/_content" ).headers["Content-Disposition"] == 'attachment; filename=bar.jpg' f.response.content = b"" - assert self.fetch("/flows/42/response/content").code == 400 + assert self.fetch("/flows/42/response/_content").code == 400 f.revert() def test_update_flow_content(self): assert self.fetch( - "/flows/42/request/content", + "/flows/42/request/_content", method="POST", body="new" ).code == 200 @@ -222,7 +222,7 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): b'--somefancyboundary--\r\n' ) assert self.fetch( - "/flows/42/request/content", + "/flows/42/request/_content", method="POST", headers={"Content-Type": 'multipart/form-data; boundary="somefancyboundary"'}, body=body diff --git a/web/src/js/__tests__/components/ContentView/__snapshots__/ContentViewOptionsSpec.js.snap b/web/src/js/__tests__/components/ContentView/__snapshots__/ContentViewOptionsSpec.js.snap index e3561ec1a..01fab0a7e 100644 --- a/web/src/js/__tests__/components/ContentView/__snapshots__/ContentViewOptionsSpec.js.snap +++ b/web/src/js/__tests__/components/ContentView/__snapshots__/ContentViewOptionsSpec.js.snap @@ -13,7 +13,7 @@ exports[`ContentViewOptions Component should render correctly 1`] = `   `; diff --git a/web/src/js/__tests__/components/ContentView/__snapshots__/DownloadContentButtonSpec.js.snap b/web/src/js/__tests__/components/ContentView/__snapshots__/DownloadContentButtonSpec.js.snap index 66900ca4e..4c578a0c2 100644 --- a/web/src/js/__tests__/components/ContentView/__snapshots__/DownloadContentButtonSpec.js.snap +++ b/web/src/js/__tests__/components/ContentView/__snapshots__/DownloadContentButtonSpec.js.snap @@ -3,7 +3,7 @@ exports[`DownloadContentButton Component should render correctly 1`] = ` { let msg = "foo", view = "bar", flow = { request: msg, id: 1} expect(utils.MessageUtils.getContentURL(flow, msg, view)).toEqual( - "/flows/1/request/content/bar" + "/flows/1/request/content/bar.json" ) expect(utils.MessageUtils.getContentURL(flow, msg, '')).toEqual( - "/flows/1/request/content" + "/flows/1/request/_content" ) // response flow = {response: msg, id: 2} expect(utils.MessageUtils.getContentURL(flow, msg, view)).toEqual( - "/flows/2/response/content/bar" + "/flows/2/response/content/bar.json" ) }) }) diff --git a/web/src/js/backends/static.js b/web/src/js/backends/static.js index 6657fecf1..7e87a5eda 100644 --- a/web/src/js/backends/static.js +++ b/web/src/js/backends/static.js @@ -11,10 +11,8 @@ export default class StaticBackend { } onOpen() { - this.fetchData("settings") this.fetchData("flows") - this.fetchData("events") - this.fetchData("options") + // this.fetchData("events") # TODO: Add events log to static viewer. } fetchData(resource) { diff --git a/web/src/js/components/FlowView/Messages.jsx b/web/src/js/components/FlowView/Messages.jsx index 4a31faf4d..b69dfb69d 100644 --- a/web/src/js/components/FlowView/Messages.jsx +++ b/web/src/js/components/FlowView/Messages.jsx @@ -9,6 +9,7 @@ import ContentView from '../ContentView' import ContentViewOptions from '../ContentView/ContentViewOptions' import ValidateEditor from '../ValueEditor/ValidateEditor' import ValueEditor from '../ValueEditor/ValueEditor' +import HideInStatic from '../common/HideInStatic' import Headers from './Headers' import { startEdit, updateEdit } from '../../ducks/ui/flow' @@ -105,6 +106,7 @@ export class Request extends Component { onContentChange={content => updateFlow({ request: {content}})} message={flow.request}/> + {!noContent &&
uploadContent(flow, content, "request")}/>
} + ) } @@ -172,6 +175,7 @@ export class Response extends Component { message={flow.response} /> + {!noContent && } + ) } diff --git a/web/src/js/flow/utils.js b/web/src/js/flow/utils.js index 3c38058ee..9915a639a 100644 --- a/web/src/js/flow/utils.js +++ b/web/src/js/flow/utils.js @@ -49,7 +49,7 @@ export var MessageUtils = { } else if (message === flow.response) { message = "response"; } - return `/flows/${flow.id}/${message}/content` + (view ? `/${view}` : ''); + return `/flows/${flow.id}/${message}/` + (view ? `content/${view}.json` : '_content'); } }; diff --git a/web/src/js/utils.js b/web/src/js/utils.js index e8470cec4..3aeba1b19 100644 --- a/web/src/js/utils.js +++ b/web/src/js/utils.js @@ -88,6 +88,8 @@ export function fetchApi(url, options={}) { } else { url += "&" + xsrf; } + } else { + url += '.json' } return fetch(url, {