Merge pull request #2510 from MatthewShao/static-viewer

[WIP][web]Static viewer converter for mitmweb

mitmproxy/addons/static_viewer.py

@@ -0,0 +1,99 @@
+import json
+import os.path
+import pathlib
+import shutil
+import time
+import typing
+
+from mitmproxy import contentviews
+from mitmproxy import ctx
+from mitmproxy import flowfilter
+from mitmproxy import io, flow
+from mitmproxy.tools.web.app import flow_to_json
+
+web_dir = pathlib.Path(__file__).absolute().parent.parent / "tools" / "web"
+
+
+def save_static(path: pathlib.Path) -> None:
+    """
+    Save the files for the static web view.
+    """
+    # We want to overwrite the static files to keep track of the update.
+    if (path / "static").exists():
+        shutil.rmtree(str(path / "static"))
+    shutil.copytree(str(web_dir / "static"), str(path / "static"))
+    shutil.copyfile(str(web_dir / 'templates' / 'index.html'), str(path / "index.html"))
+
+    with open(str(path / "static" / "static.js"), "w") as f:
+        f.write("MITMWEB_STATIC = true;")
+
+
+def save_filter_help(path: pathlib.Path) -> None:
+    with open(str(path / 'filter-help.json'), 'w') as f:
+        json.dump(dict(commands=flowfilter.help), f)
+
+
+def save_flows(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None:
+    with open(str(path / 'flows.json'), 'w') as f:
+        json.dump(
+            [flow_to_json(f) for f in flows],
+            f
+        )
+
+
+def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None:
+    for f in flows:
+        for m in ('request', 'response'):
+            message = getattr(f, m)
+            message_path = path / "flows" / f.id / m
+            os.makedirs(str(message_path / "content"), exist_ok=True)
+
+            with open(str(message_path / '_content'), 'wb') as content_file:
+                # don't use raw_content here as this is served with a default content type
+                if message:
+                    content_file.write(message.content)
+                else:
+                    content_file.write(b'No content.')
+
+            # content_view
+            t = time.time()
+            if message:
+                description, lines, error = contentviews.get_message_content_view(
+                    'Auto', message
+                )
+            else:
+                description, lines = 'No content.', []
+            if time.time() - t > 0.1:
+                ctx.log(
+                    "Slow content view: {} took {}s".format(
+                        description.strip(),
+                        round(time.time() - t, 1)
+                    ),
+                    "info"
+                )
+            with open(str(message_path / "content" / "Auto.json"), "w") as content_view_file:
+                json.dump(
+                    dict(lines=list(lines), description=description),
+                    content_view_file
+                )
+
+
+class StaticViewer:
+    # TODO: make this a command at some point.
+    def load(self, loader):
+        loader.add_option(
+            "web_static_viewer", typing.Optional[str], "",
+            "The path to output a static viewer."
+        )
+
+    def configure(self, updated):
+        if "web_static_viewer" in updated and ctx.options.web_static_viewer:
+            flows = io.read_flows_from_paths([ctx.options.rfile])
+            p = pathlib.Path(ctx.options.web_static_viewer).expanduser()
+            self.export(p, flows)
+
+    def export(self, path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None:
+        save_static(path)
+        save_filter_help(path)
+        save_flows(path, flows)
+        save_flows_content(path, flows)

mitmproxy/tools/web/app.py

@@ -5,7 +5,6 @@ import os.path
 import re
 from io import BytesIO
 
-import mitmproxy.addons.view
 import mitmproxy.flow
 import tornado.escape
 import tornado.web
@@ -149,7 +148,7 @@ class RequestHandler(tornado.web.RequestHandler):
             return self.request.body
 
     @property
-    def view(self) -> mitmproxy.addons.view.View:
+    def view(self) -> "mitmproxy.addons.view.View":
         return self.application.master.view
 
     @property
@@ -466,10 +465,10 @@ class Application(tornado.web.Application):
         self.master = master
         handlers = [
             (r"/", IndexHandler),
-            (r"/filter-help", FilterHelp),
+            (r"/filter-help(?:\.json)?", FilterHelp),
             (r"/updates", ClientConnection),
-            (r"/events", Events),
+            (r"/events(?:\.json)?", Events),
-            (r"/flows", Flows),
+            (r"/flows(?:\.json)?", Flows),
             (r"/flows/dump", DumpFlows),
             (r"/flows/resume", ResumeFlows),
             (r"/flows/kill", KillFlows),
@@ -479,13 +478,13 @@ class Application(tornado.web.Application):
             (r"/flows/(?P<flow_id>[0-9a-f\-]+)/duplicate", DuplicateFlow),
             (r"/flows/(?P<flow_id>[0-9a-f\-]+)/replay", ReplayFlow),
             (r"/flows/(?P<flow_id>[0-9a-f\-]+)/revert", RevertFlow),
-            (r"/flows/(?P<flow_id>[0-9a-f\-]+)/(?P<message>request|response)/content", FlowContent),
+            (r"/flows/(?P<flow_id>[0-9a-f\-]+)/(?P<message>request|response)/_content", FlowContent),
             (
-                r"/flows/(?P<flow_id>[0-9a-f\-]+)/(?P<message>request|response)/content/(?P<content_view>[0-9a-zA-Z\-\_]+)",
+                r"/flows/(?P<flow_id>[0-9a-f\-]+)/(?P<message>request|response)/content/(?P<content_view>[0-9a-zA-Z\-\_]+)(?:\.json)?",
                 FlowContentView),
-            (r"/settings", Settings),
+            (r"/settings(?:\.json)?", Settings),
             (r"/clear", ClearAll),
-            (r"/options", Options),
+            (r"/options(?:\.json)?", Options),
             (r"/options/save", SaveOptions)
         ]
         settings = dict(

mitmproxy/tools/web/master.py

@@ -12,6 +12,7 @@ from mitmproxy.addons import readfile
 from mitmproxy.addons import termlog
 from mitmproxy.addons import view
 from mitmproxy.addons import termstatus
+from mitmproxy.addons import static_viewer
 from mitmproxy.options import Options  # noqa
 from mitmproxy.tools.web import app, webaddons
 
@@ -37,6 +38,7 @@ class WebMaster(master.Master):
             webaddons.WebAddon(),
             intercept.Intercept(),
             readfile.ReadFile(),
+            static_viewer.StaticViewer(),
             self.view,
             self.events,
         )

mitmproxy/tools/web/templates/index.html

@@ -7,6 +7,7 @@
     <link rel="stylesheet" href="/static/vendor.css"/>
     <link rel="stylesheet" href="/static/app.css"/>
     <link rel="icon" href="/static/images/favicon.ico" type="image/x-icon"/>
+    <script src="/static/static.js"></script>
     <script src="/static/vendor.js"></script>
     <script src="/static/app.js"></script>
 </head>

test/mitmproxy/addons/test_static_viewer.py

@@ -0,0 +1,63 @@
+import json
+from unittest import mock
+
+from mitmproxy.test import taddons
+from mitmproxy.test import tflow
+
+from mitmproxy import flowfilter
+from mitmproxy.tools.web.app import flow_to_json
+
+from mitmproxy.addons import static_viewer
+from mitmproxy.addons import save
+
+
+def test_save_static(tmpdir):
+    tmpdir.mkdir('static')
+    static_viewer.save_static(tmpdir)
+    assert len(tmpdir.listdir()) == 2
+    assert tmpdir.join('index.html').check(file=1)
+    assert tmpdir.join('static/static.js').read() == 'MITMWEB_STATIC = true;'
+
+
+def test_save_filter_help(tmpdir):
+    static_viewer.save_filter_help(tmpdir)
+    f = tmpdir.join('/filter-help.json')
+    assert f.check(file=1)
+    assert f.read() == json.dumps(dict(commands=flowfilter.help))
+
+
+def test_save_flows(tmpdir):
+    flows = [tflow.tflow(req=True, resp=None), tflow.tflow(req=True, resp=True)]
+    static_viewer.save_flows(tmpdir, flows)
+    assert tmpdir.join('flows.json').check(file=1)
+    assert tmpdir.join('flows.json').read() == json.dumps([flow_to_json(f) for f in flows])
+
+
+@mock.patch('mitmproxy.ctx.log')
+def test_save_flows_content(ctx, tmpdir):
+    flows = [tflow.tflow(req=True, resp=None), tflow.tflow(req=True, resp=True)]
+    with mock.patch('time.time', mock.Mock(side_effect=[1, 2, 2] * 4)):
+        static_viewer.save_flows_content(tmpdir, flows)
+    flows_path = tmpdir.join('flows')
+    assert len(flows_path.listdir()) == len(flows)
+    for p in flows_path.listdir():
+        assert p.join('request').check(dir=1)
+        assert p.join('response').check(dir=1)
+        assert p.join('request/_content').check(file=1)
+        assert p.join('request/content').check(dir=1)
+        assert p.join('response/_content').check(file=1)
+        assert p.join('response/content').check(dir=1)
+        assert p.join('request/content/Auto.json').check(file=1)
+        assert p.join('response/content/Auto.json').check(file=1)
+
+
+def test_static_viewer(tmpdir):
+    s = static_viewer.StaticViewer()
+    sa = save.Save()
+    with taddons.context() as tctx:
+        sa.save([tflow.tflow(resp=True)], str(tmpdir.join('foo')))
+        tctx.master.addons.add(s)
+        tctx.configure(s, web_static_viewer=str(tmpdir), rfile=str(tmpdir.join('foo')))
+        assert tmpdir.join('index.html').check(file=1)
+        assert tmpdir.join('static').check(dir=1)
+        assert tmpdir.join('flows').check(dir=1)

test/mitmproxy/tools/web/test_app.py

@@ -186,7 +186,7 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
         f.response.headers["Content-Encoding"] = "ran\x00dom"
         f.response.headers["Content-Disposition"] = 'inline; filename="filename.jpg"'
 
-        r = self.fetch("/flows/42/response/content")
+        r = self.fetch("/flows/42/response/_content")
         assert r.body == b"message"
         assert r.headers["Content-Encoding"] == "random"
         assert r.headers["Content-Disposition"] == 'attachment; filename="filename.jpg"'
@@ -194,17 +194,17 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
         del f.response.headers["Content-Disposition"]
         f.request.path = "/foo/bar.jpg"
         assert self.fetch(
-            "/flows/42/response/content"
+            "/flows/42/response/_content"
         ).headers["Content-Disposition"] == 'attachment; filename=bar.jpg'
 
         f.response.content = b""
-        assert self.fetch("/flows/42/response/content").code == 400
+        assert self.fetch("/flows/42/response/_content").code == 400
 
         f.revert()
 
     def test_update_flow_content(self):
         assert self.fetch(
-            "/flows/42/request/content",
+            "/flows/42/request/_content",
             method="POST",
             body="new"
         ).code == 200
@@ -222,7 +222,7 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
             b'--somefancyboundary--\r\n'
         )
         assert self.fetch(
-            "/flows/42/request/content",
+            "/flows/42/request/_content",
             method="POST",
             headers={"Content-Type": 'multipart/form-data; boundary="somefancyboundary"'},
             body=body

web/src/js/__tests__/components/ContentView/__snapshots__/ContentViewOptionsSpec.js.snap

@@ -13,7 +13,7 @@ exports[`ContentViewOptions Component should render correctly 1`] = `
    
   <a
     className="btn btn-default btn-xs"
-    href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/content"
+    href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/_content"
     title="Download the content of the flow."
   >
     <i

web/src/js/__tests__/components/ContentView/__snapshots__/ContentViewSpec.js.snap

@@ -17,7 +17,7 @@ exports[`ViewImage Component should render correctly 1`] = `
   <img
     alt="preview"
     className="img-thumbnail"
-    src="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/content"
+    src="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/_content"
   />
 </div>
 `;

web/src/js/__tests__/components/ContentView/__snapshots__/DownloadContentButtonSpec.js.snap

@@ -3,7 +3,7 @@
 exports[`DownloadContentButton Component should render correctly 1`] = `
 <a
   className="btn btn-default btn-xs"
-  href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/content"
+  href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/_content"
   title="Download the content of the flow."
 >
   <i

web/src/js/__tests__/components/ContentView/__snapshots__/MetaViewsSpec.js.snap

@@ -54,7 +54,7 @@ exports[`ContentTooLarge Components should render correctly 1`] = `
      
     <a
       className="btn btn-default btn-xs"
-      href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/content"
+      href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/_content"
       title="Download the content of the flow."
     >
       <i

web/src/js/__tests__/components/FlowView/__snapshots__/MessagesSpec.js.snap

@@ -265,7 +265,7 @@ exports[`Request Component should render correctly 1`] = `
        
       <a
         className="btn btn-default btn-xs"
-        href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/request/content"
+        href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/request/_content"
         title="Download the content of the flow."
       >
         <i
@@ -528,7 +528,7 @@ exports[`Response Component should render correctly 1`] = `
        
       <a
         className="btn btn-default btn-xs"
-        href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/content"
+        href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/_content"
         title="Download the content of the flow."
       >
         <i

web/src/js/__tests__/components/__snapshots__/ContentViewSpec.js.snap

@@ -49,7 +49,7 @@ exports[`ContentView Component should render correctly with content too large 1`
      
     <a
       className="btn btn-default btn-xs"
-      href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/content"
+      href="/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/_content"
       title="Download the content of the flow."
     >
       <i

web/src/js/__tests__/flow/utilsSpec.js

@@ -25,15 +25,15 @@ describe('MessageUtils', () => {
         let msg = "foo", view = "bar",
             flow = { request: msg, id: 1}
         expect(utils.MessageUtils.getContentURL(flow, msg, view)).toEqual(
-            "/flows/1/request/content/bar"
+            "/flows/1/request/content/bar.json"
         )
         expect(utils.MessageUtils.getContentURL(flow, msg, '')).toEqual(
-            "/flows/1/request/content"
+            "/flows/1/request/_content"
         )
         // response
         flow = {response: msg, id: 2}
         expect(utils.MessageUtils.getContentURL(flow, msg, view)).toEqual(
-            "/flows/2/response/content/bar"
+            "/flows/2/response/content/bar.json"
         )
     })
 })

web/src/js/backends/static.js

@@ -11,10 +11,8 @@ export default class StaticBackend {
    }
 
    onOpen() {
-        this.fetchData("settings")
         this.fetchData("flows")
-        this.fetchData("events")
+        // this.fetchData("events") # TODO: Add events log to static viewer.
-        this.fetchData("options")
    }
 
    fetchData(resource) {

web/src/js/components/FlowView/Messages.jsx

@@ -9,6 +9,7 @@ import ContentView from '../ContentView'
 import ContentViewOptions from '../ContentView/ContentViewOptions'
 import ValidateEditor from '../ValueEditor/ValidateEditor'
 import ValueEditor from '../ValueEditor/ValueEditor'
+import HideInStatic from '../common/HideInStatic'
 
 import Headers from './Headers'
 import { startEdit, updateEdit } from '../../ducks/ui/flow'
@@ -105,6 +106,7 @@ export class Request extends Component {
                         onContentChange={content => updateFlow({ request: {content}})}
                         message={flow.request}/>
                 </article>
+                <HideInStatic>
                 {!noContent &&
                     <footer>
                         <ContentViewOptions
@@ -114,6 +116,7 @@ export class Request extends Component {
                             uploadContent={content => uploadContent(flow, content, "request")}/>
                     </footer>
                 }
+                </HideInStatic>
             </section>
         )
     }
@@ -172,6 +175,7 @@ export class Response extends Component {
                         message={flow.response}
                     />
                 </article>
+                <HideInStatic>
                 {!noContent &&
                     <footer >
                         <ContentViewOptions
@@ -181,6 +185,7 @@ export class Response extends Component {
                             readonly={!isEdit}/>
                     </footer>
                 }
+                </HideInStatic>
             </section>
         )
     }

web/src/js/flow/utils.js

@@ -49,7 +49,7 @@ export var MessageUtils = {
         } else if (message === flow.response) {
             message = "response";
         }
-        return `/flows/${flow.id}/${message}/content` +  (view ? `/${view}` : '');
+        return `/flows/${flow.id}/${message}/` + (view ? `content/${view}.json` : '_content');
     }
 };
 

web/src/js/utils.js

@@ -88,6 +88,8 @@ export function fetchApi(url, options={}) {
         } else {
             url += "&" + xsrf;
         }
+    } else {
+        url += '.json'
     }
 
     return fetch(url, {