From fea6041cde7c9642007d852272573bfe97c360a3 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 22 Nov 2016 23:53:17 +0100 Subject: [PATCH 1/5] mitmweb: cleanup, reimplement eventlog --- mitmproxy/addons/eventstore.py | 19 ++++++ mitmproxy/tools/main.py | 1 - mitmproxy/tools/web/app.py | 88 +++++++++---------------- mitmproxy/tools/web/master.py | 116 +++++++++++---------------------- 4 files changed, 87 insertions(+), 137 deletions(-) create mode 100644 mitmproxy/addons/eventstore.py diff --git a/mitmproxy/addons/eventstore.py b/mitmproxy/addons/eventstore.py new file mode 100644 index 000000000..0afd018a6 --- /dev/null +++ b/mitmproxy/addons/eventstore.py @@ -0,0 +1,19 @@ +from typing import List # noqa + +import blinker +from mitmproxy import log + + +class EventStore: + def __init__(self): + self.data = [] # type: List[log.LogEntry] + self.sig_add = blinker.Signal() + self.sig_refresh = blinker.Signal() + + def log(self, entry: log.LogEntry): + self.data.append(entry) + self.sig_add.send(self, entry=entry) + + def clear(self): + self.data.clear() + self.sig_refresh.send(self) diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 69dd3791b..478690eb2 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -135,7 +135,6 @@ def mitmweb(args=None): # pragma: no cover web_options.wdebug = args.wdebug web_options.wiface = args.wiface web_options.wport = args.wport - web_options.process_web_options(parser) server = process_options(parser, web_options, args) m = web.master.WebMaster(web_options, server) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 25a46169f..8f71d8dac 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -1,5 +1,3 @@ - -import base64 import hashlib import json import logging @@ -7,19 +5,20 @@ import os.path import re from io import BytesIO +import mitmproxy.addons.view +import mitmproxy.flow +import tornado.escape import tornado.web import tornado.websocket -import tornado.escape from mitmproxy import contentviews from mitmproxy import flowfilter from mitmproxy import http from mitmproxy import io +from mitmproxy import log from mitmproxy import version -import mitmproxy.addons.view -import mitmproxy.flow -def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict: +def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: """ Remove flow message content and cert to save transmission space. @@ -46,8 +45,10 @@ def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict: "path": flow.request.path, "http_version": flow.request.http_version, "headers": tuple(flow.request.headers.items(True)), - "contentLength": len(flow.request.raw_content) if flow.request.raw_content is not None else None, - "contentHash": hashlib.sha256(flow.request.raw_content).hexdigest() if flow.request.raw_content is not None else None, + "contentLength": len( + flow.request.raw_content) if flow.request.raw_content is not None else None, + "contentHash": hashlib.sha256( + flow.request.raw_content).hexdigest() if flow.request.raw_content is not None else None, "timestamp_start": flow.request.timestamp_start, "timestamp_end": flow.request.timestamp_end, "is_replay": flow.request.is_replay, @@ -58,8 +59,10 @@ def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict: "status_code": flow.response.status_code, "reason": flow.response.reason, "headers": tuple(flow.response.headers.items(True)), - "contentLength": len(flow.response.raw_content) if flow.response.raw_content is not None else None, - "contentHash": hashlib.sha256(flow.response.raw_content).hexdigest() if flow.response.raw_content is not None else None, + "contentLength": len( + flow.response.raw_content) if flow.response.raw_content is not None else None, + "contentHash": hashlib.sha256( + flow.response.raw_content).hexdigest() if flow.response.raw_content is not None else None, "timestamp_start": flow.response.timestamp_start, "timestamp_end": flow.response.timestamp_end, "is_replay": flow.response.is_replay, @@ -69,34 +72,18 @@ def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict: return f +def logentry_to_json(e: log.LogEntry): + return { + "message": e.msg, + "level": e.level + } + + class APIError(tornado.web.HTTPError): pass -class BasicAuth: - - def set_auth_headers(self): - self.set_status(401) - self.set_header('WWW-Authenticate', 'Basic realm=MITMWeb') - self._transforms = [] - self.finish() - - def prepare(self): - wauthenticator = self.application.settings['wauthenticator'] - if wauthenticator: - auth_header = self.request.headers.get('Authorization') - if auth_header is None or not auth_header.startswith('Basic '): - self.set_auth_headers() - else: - auth_decoded = base64.decodebytes(auth_header[6:]) - username, password = auth_decoded.split(':', 2) - if not wauthenticator.test(username, password): - self.set_auth_headers() - raise APIError(401, "Invalid username or password.") - - -class RequestHandler(BasicAuth, tornado.web.RequestHandler): - +class RequestHandler(tornado.web.RequestHandler): def write(self, chunk): # Writing arrays on the top level is ok nowadays. # http://flask.pocoo.org/docs/0.11/security/#json-security @@ -150,7 +137,6 @@ class RequestHandler(BasicAuth, tornado.web.RequestHandler): class IndexHandler(RequestHandler): - def get(self): token = self.xsrf_token # https://github.com/tornadoweb/tornado/issues/645 assert token @@ -158,14 +144,13 @@ class IndexHandler(RequestHandler): class FilterHelp(RequestHandler): - def get(self): self.write(dict( commands=flowfilter.help )) -class WebSocketEventBroadcaster(BasicAuth, tornado.websocket.WebSocketHandler): +class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler): # raise an error if inherited class doesn't specify its own instance. connections = None # type: set @@ -191,9 +176,8 @@ class ClientConnection(WebSocketEventBroadcaster): class Flows(RequestHandler): - def get(self): - self.write([convert_flow_to_json_dict(f) for f in self.view]) + self.write([flow_to_json(f) for f in self.view]) class DumpFlows(RequestHandler): @@ -219,25 +203,22 @@ class DumpFlows(RequestHandler): class ClearAll(RequestHandler): - def post(self): self.view.clear() + self.master.events.clear() class AcceptFlows(RequestHandler): - def post(self): self.master.accept_all(self.master) class AcceptFlow(RequestHandler): - def post(self, flow_id): self.flow.resume(self.master) class FlowHandler(RequestHandler): - def delete(self, flow_id): if self.flow.killable: self.flow.kill(self.master) @@ -286,19 +267,16 @@ class FlowHandler(RequestHandler): class DuplicateFlow(RequestHandler): - def post(self, flow_id): self.master.view.duplicate_flow(self.flow) class RevertFlow(RequestHandler): - def post(self, flow_id): self.flow.revert() class ReplayFlow(RequestHandler): - def post(self, flow_id): self.flow.backup() self.flow.response = None @@ -310,7 +288,6 @@ class ReplayFlow(RequestHandler): class FlowContent(RequestHandler): - def post(self, flow_id, message): self.flow.backup() message = getattr(self.flow, message) @@ -347,15 +324,14 @@ class FlowContent(RequestHandler): class FlowContentView(RequestHandler): - def get(self, flow_id, message, content_view): message = getattr(self.flow, message) description, lines, error = contentviews.get_message_content_view( content_view.replace('_', ' '), message ) -# if error: -# add event log + # if error: + # add event log self.write(dict( lines=list(lines), @@ -364,13 +340,11 @@ class FlowContentView(RequestHandler): class Events(RequestHandler): - def get(self): - self.write([]) # FIXME + self.write([logentry_to_json(e) for e in self.master.events.data]) class Settings(RequestHandler): - def get(self): self.write(dict( version=version.VERSION, @@ -432,8 +406,7 @@ class Settings(RequestHandler): class Application(tornado.web.Application): - - def __init__(self, master, debug, wauthenticator): + def __init__(self, master, debug): self.master = master handlers = [ (r"/", IndexHandler), @@ -449,7 +422,9 @@ class Application(tornado.web.Application): (r"/flows/(?P[0-9a-f\-]+)/replay", ReplayFlow), (r"/flows/(?P[0-9a-f\-]+)/revert", RevertFlow), (r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/content", FlowContent), - (r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/content/(?P[0-9a-zA-Z\-\_]+)", FlowContentView), + ( + r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/content/(?P[0-9a-zA-Z\-\_]+)", + FlowContentView), (r"/settings", Settings), (r"/clear", ClearAll), ] @@ -460,6 +435,5 @@ class Application(tornado.web.Application): cookie_secret=os.urandom(256), debug=debug, autoreload=False, - wauthenticator=wauthenticator, ) super().__init__(handlers, **settings) diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index d2203f10c..453455f03 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -1,49 +1,20 @@ import sys import webbrowser +from typing import Optional import tornado.httpserver import tornado.ioloop - -from typing import Optional - from mitmproxy import addons from mitmproxy import exceptions -from mitmproxy.addons import view -from mitmproxy.addons import intercept -from mitmproxy import options +from mitmproxy import log from mitmproxy import master +from mitmproxy import options +from mitmproxy.addons import eventstore +from mitmproxy.addons import intercept +from mitmproxy.addons import view from mitmproxy.tools.web import app -class Stop(Exception): - pass - - -class _WebState(): - def add_log(self, e, level): - # server-side log ids are odd - self._last_event_id += 2 - entry = { - "id": self._last_event_id, - "message": e, - "level": level - } - self.events.append(entry) - app.ClientConnection.broadcast( - resource="events", - cmd="add", - data=entry - ) - - def clear(self): - super().clear() - self.events.clear() - app.ClientConnection.broadcast( - resource="events", - cmd="reset" - ) - - class Options(options.Options): def __init__( self, @@ -52,54 +23,32 @@ class Options(options.Options): wdebug: bool = False, wport: int = 8081, wiface: str = "127.0.0.1", - # wauthenticator: Optional[authentication.PassMan] = None, - wsingleuser: Optional[str] = None, - whtpasswd: Optional[str] = None, **kwargs ) -> None: + self.intercept = intercept self.wdebug = wdebug self.wport = wport self.wiface = wiface - # self.wauthenticator = wauthenticator - # self.wsingleuser = wsingleuser - # self.whtpasswd = whtpasswd - self.intercept = intercept super().__init__(**kwargs) - # TODO: This doesn't belong here. - def process_web_options(self, parser): - # if self.wsingleuser or self.whtpasswd: - # if self.wsingleuser: - # if len(self.wsingleuser.split(':')) != 2: - # return parser.error( - # "Invalid single-user specification. Please use the format username:password" - # ) - # username, password = self.wsingleuser.split(':') - # # self.wauthenticator = authentication.PassManSingleUser(username, password) - # elif self.whtpasswd: - # try: - # self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd) - # except ValueError as v: - # return parser.error(v.message) - # else: - # self.wauthenticator = None - pass - class WebMaster(master.Master): - def __init__(self, options, server): super().__init__(options, server) self.view = view.View() - self.view.sig_view_add.connect(self._sig_add) - self.view.sig_view_remove.connect(self._sig_remove) - self.view.sig_view_update.connect(self._sig_update) - self.view.sig_view_refresh.connect(self._sig_refresh) + self.view.sig_view_add.connect(self._sig_view_add) + self.view.sig_view_remove.connect(self._sig_view_remove) + self.view.sig_view_update.connect(self._sig_view_update) + self.view.sig_view_refresh.connect(self._sig_view_refresh) + + self.events = eventstore.EventStore() + self.events.sig_add.connect(self._sig_events_add) + self.events.sig_refresh.connect(self._sig_events_refresh) self.addons.add(*addons.default_addons()) - self.addons.add(self.view, intercept.Intercept()) + self.addons.add(self.view, self.events, intercept.Intercept()) self.app = app.Application( - self, self.options.wdebug, False + self, self.options.wdebug ) # This line is just for type hinting self.options = self.options # type: Options @@ -112,33 +61,46 @@ class WebMaster(master.Master): "error" ) - def _sig_add(self, view, flow): + def _sig_view_add(self, view, flow): app.ClientConnection.broadcast( resource="flows", cmd="add", - data=app.convert_flow_to_json_dict(flow) + data=app.flow_to_json(flow) ) - def _sig_update(self, view, flow): + def _sig_view_update(self, view, flow): app.ClientConnection.broadcast( resource="flows", cmd="update", - data=app.convert_flow_to_json_dict(flow) + data=app.flow_to_json(flow) ) - def _sig_remove(self, view, flow): + def _sig_view_remove(self, view, flow): app.ClientConnection.broadcast( resource="flows", cmd="remove", data=dict(id=flow.id) ) - def _sig_refresh(self, view): + def _sig_view_refresh(self, view): app.ClientConnection.broadcast( resource="flows", cmd="reset" ) + def _sig_events_add(self, event_store, entry: log.LogEntry): + app.ClientConnection.broadcast( + resource="events", + cmd="add", + data=app.logentry_to_json(entry) + ) + + def _sig_events_refresh(self, event_store): + app.ClientConnection.broadcast( + resource="events", + cmd="reset" + ) + def run(self): # pragma: no cover iol = tornado.ioloop.IOLoop.instance() @@ -155,13 +117,9 @@ class WebMaster(master.Master): print("No webbrowser found. Please open a browser and point it to {}".format(url)) iol.start() - except (Stop, KeyboardInterrupt): + except (KeyboardInterrupt): self.shutdown() - # def add_log(self, e, level="info"): - # super().add_log(e, level) - # return self.state.add_log(e, level) - def open_browser(url: str) -> bool: """ From 72dcf70db20ef5d09004b1be1355612500a32592 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 23 Nov 2016 00:12:45 +0100 Subject: [PATCH 2/5] mitmweb: fix event display --- mitmproxy/tools/web/app.py | 3 ++- web/src/js/components/EventLog.jsx | 2 +- web/src/js/ducks/eventLog.js | 4 +--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 8f71d8dac..fa0b702f6 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -72,8 +72,9 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: return f -def logentry_to_json(e: log.LogEntry): +def logentry_to_json(e: log.LogEntry) -> dict: return { + "id": id(e), # we just need some kind of id. "message": e.msg, "level": e.level } diff --git a/web/src/js/components/EventLog.jsx b/web/src/js/components/EventLog.jsx index 636e3e9a3..1a4495115 100644 --- a/web/src/js/components/EventLog.jsx +++ b/web/src/js/components/EventLog.jsx @@ -70,7 +70,7 @@ class EventLog extends Component { export default connect( state => ({ filters: state.eventLog.filters, - events: state.eventLog.view.data, + events: state.eventLog.view, }), { close: toggleVisibility, diff --git a/web/src/js/ducks/eventLog.js b/web/src/js/ducks/eventLog.js index 776e4b08f..73eaf2e89 100644 --- a/web/src/js/ducks/eventLog.js +++ b/web/src/js/ducks/eventLog.js @@ -49,14 +49,12 @@ export function toggleVisibility() { return { type: TOGGLE_VISIBILITY } } -let logId = 1 // client-side log ids are odd export function add(message, level = 'web') { let data = { - id: logId, + id: Math.random().toString(), message, level, } - logId += 2 return { type: ADD, cmd: "add", From f1662cbfd7e2d6064d8a1fde22b6f3d288766f04 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 23 Nov 2016 00:23:36 +0100 Subject: [PATCH 3/5] add eventstore tests --- test/mitmproxy/addons/test_evenstore.py | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 test/mitmproxy/addons/test_evenstore.py diff --git a/test/mitmproxy/addons/test_evenstore.py b/test/mitmproxy/addons/test_evenstore.py new file mode 100644 index 000000000..78eb32877 --- /dev/null +++ b/test/mitmproxy/addons/test_evenstore.py @@ -0,0 +1,32 @@ +import mock +from mitmproxy import log +from mitmproxy.addons import eventstore + + +def test_simple(): + store = eventstore.EventStore() + assert not store.data + + sig_add = mock.Mock(spec=lambda: 42) + sig_refresh = mock.Mock(spec=lambda: 42) + store.sig_add.connect(sig_add) + store.sig_refresh.connect(sig_refresh) + + assert not sig_add.called + assert not sig_refresh.called + + # test .log() + store.log(log.LogEntry("test", "info")) + assert store.data + + assert sig_add.called + assert not sig_refresh.called + + # test .clear() + sig_add.reset_mock() + + store.clear() + assert not store.data + + assert not sig_add.called + assert sig_refresh.called From dc75605e463f064fce07a1a7bf23b16f66742cbb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 23 Nov 2016 01:26:47 +0100 Subject: [PATCH 4/5] minor fixes --- mitmproxy/addons/eventstore.py | 6 +++--- web/src/js/components/ContentView/ContentViewOptions.jsx | 8 ++++---- web/src/js/components/ContentView/ContentViews.jsx | 4 +--- web/src/js/components/ContentView/ViewSelector.jsx | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/mitmproxy/addons/eventstore.py b/mitmproxy/addons/eventstore.py index 0afd018a6..4e410c984 100644 --- a/mitmproxy/addons/eventstore.py +++ b/mitmproxy/addons/eventstore.py @@ -1,16 +1,16 @@ from typing import List # noqa import blinker -from mitmproxy import log +from mitmproxy.log import LogEntry class EventStore: def __init__(self): - self.data = [] # type: List[log.LogEntry] + self.data = [] # type: List[LogEntry] self.sig_add = blinker.Signal() self.sig_refresh = blinker.Signal() - def log(self, entry: log.LogEntry): + def log(self, entry: LogEntry): self.data.append(entry) self.sig_add.send(self, entry=entry) diff --git a/web/src/js/components/ContentView/ContentViewOptions.jsx b/web/src/js/components/ContentView/ContentViewOptions.jsx index fed3a088b..6bc66db22 100644 --- a/web/src/js/components/ContentView/ContentViewOptions.jsx +++ b/web/src/js/components/ContentView/ContentViewOptions.jsx @@ -9,15 +9,14 @@ ContentViewOptions.propTypes = { message: React.PropTypes.object.isRequired, } -function ContentViewOptions(props) { - const { flow, message, uploadContent, readonly, contentViewDescription } = props +function ContentViewOptions({ flow, message, uploadContent, readonly, contentViewDescription }) { return (
    - + {!readonly && }   {contentViewDescription}
@@ -26,6 +25,7 @@ function ContentViewOptions(props) { export default connect( state => ({ - contentViewDescription: state.ui.flow.viewDescription + contentViewDescription: state.ui.flow.viewDescription, + readonly: !state.ui.flow.modifiedFlow, }) )(ContentViewOptions) diff --git a/web/src/js/components/ContentView/ContentViews.jsx b/web/src/js/components/ContentView/ContentViews.jsx index 32a075642..db2391950 100644 --- a/web/src/js/components/ContentView/ContentViews.jsx +++ b/web/src/js/components/ContentView/ContentViews.jsx @@ -63,6 +63,7 @@ class ViewServer extends Component { let lines = this.props.showFullContent ? this.data.lines : this.data.lines.slice(0, maxLines) return (
+ {ViewImage.matches(message) && } 
                     {lines.map((line, i) =>
                         

@@ -77,9 +78,6 @@ class ViewServer extends Component {
                         

                     )}
- {ViewImage.matches(message) && - - }
) } diff --git a/web/src/js/components/ContentView/ViewSelector.jsx b/web/src/js/components/ContentView/ViewSelector.jsx index ab433ea3b..fcdc3ee34 100644 --- a/web/src/js/components/ContentView/ViewSelector.jsx +++ b/web/src/js/components/ContentView/ViewSelector.jsx @@ -14,7 +14,7 @@ ViewSelector.propTypes = { function ViewSelector ({contentViews, activeView, isEdit, setContentView}){ let edit = ContentViews.Edit.displayName - let inner = View: {activeView} + let inner = View: {activeView} return ( From 45332006a3da246679e6043b4abee06cd3ba0636 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 23 Nov 2016 22:35:07 +0100 Subject: [PATCH 5/5] mitmweb: 100% app test coverage, numerous fixes --- examples/complex/dup_and_replay.py | 3 +- mitmproxy/addons/view.py | 7 + mitmproxy/contentviews.py | 2 +- mitmproxy/io.py | 10 +- mitmproxy/master.py | 10 +- mitmproxy/tools/console/flowlist.py | 6 +- mitmproxy/tools/console/flowview.py | 6 +- mitmproxy/tools/console/master.py | 2 +- mitmproxy/tools/web/app.py | 175 ++++++------ mitmproxy/tools/web/master.py | 9 + .../data/addonscripts/duplicate_flow.py | 6 - test/mitmproxy/test_web_app.py | 253 +++++++++++++++++- web/src/js/components/Footer.jsx | 6 +- web/src/js/components/Header/OptionMenu.jsx | 8 +- 14 files changed, 377 insertions(+), 126 deletions(-) delete mode 100644 test/mitmproxy/data/addonscripts/duplicate_flow.py diff --git a/examples/complex/dup_and_replay.py b/examples/complex/dup_and_replay.py index bf7c2a4e1..2baa1ea6d 100644 --- a/examples/complex/dup_and_replay.py +++ b/examples/complex/dup_and_replay.py @@ -2,6 +2,7 @@ from mitmproxy import ctx def request(flow): - f = ctx.master.state.duplicate_flow(flow) + f = flow.copy() + ctx.master.view.add(f) f.request.path = "/changed" ctx.master.replay_request(f, block=True) diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index e151e081f..b8b6093f5 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -280,6 +280,13 @@ class View(collections.Sequence): # The value was not in the view pass + def get_by_id(self, flow_id: str) -> typing.Optional[mitmproxy.flow.Flow]: + """ + Get flow with the given id from the store. + Returns None if the flow is not found. + """ + return self._store.get(flow_id) + # Event handlers def configure(self, opts, updated): if "filter" in updated: diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index 4f2838eef..24ecbf197 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -609,7 +609,7 @@ def get_message_content_view(viewname, message): """ viewmode = get(viewname) if not viewmode: - get("auto") + viewmode = get("auto") try: content = message.content except ValueError: diff --git a/mitmproxy/io.py b/mitmproxy/io.py index 27ffa036d..8f868e563 100644 --- a/mitmproxy/io.py +++ b/mitmproxy/io.py @@ -1,6 +1,8 @@ import os +from typing import Iterable from mitmproxy import exceptions +from mitmproxy import flow from mitmproxy import flowfilter from mitmproxy import http from mitmproxy import tcp @@ -27,7 +29,7 @@ class FlowReader: def __init__(self, fo): self.fo = fo - def stream(self): + def stream(self) -> Iterable[flow.Flow]: """ Yields Flow objects from the dump. """ @@ -52,10 +54,10 @@ class FilteredFlowWriter: self.fo = fo self.flt = flt - def add(self, flow): - if self.flt and not flowfilter.match(self.flt, flow): + def add(self, f: flow.Flow): + if self.flt and not flowfilter.match(self.flt, f): return - d = flow.get_state() + d = f.get_state() tnetstring.dump(d, self.fo) diff --git a/mitmproxy/master.py b/mitmproxy/master.py index 55eb74e51..c18e18caa 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -156,7 +156,7 @@ class Master: for e, o in events.event_sequence(f): getattr(self, e)(o) - def load_flows(self, fr): + def load_flows(self, fr: io.FlowReader) -> int: """ Load flows from a FlowReader object. """ @@ -166,7 +166,7 @@ class Master: self.load_flow(i) return cnt - def load_flows_file(self, path): + def load_flows_file(self, path: str) -> int: path = os.path.expanduser(path) try: if path == "-": @@ -180,7 +180,11 @@ class Master: except IOError as v: raise exceptions.FlowReadException(v.strerror) - def replay_request(self, f, block=False): + def replay_request( + self, + f: http.HTTPFlow, + block: bool=False + ) -> http_replay.RequestReplayThread: """ Replay a HTTP request to receive a new response from the server. diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index 395a9d52b..d7c312e5e 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -303,8 +303,8 @@ class FlowListWalker(urwid.ListWalker): class FlowListBox(urwid.ListBox): - def __init__(self, master): - self.master = master + def __init__(self, master: "mitmproxy.tools.console.master.ConsoleMaster"): + self.master = master # type: "mitmproxy.tools.console.master.ConsoleMaster" super().__init__(FlowListWalker(master)) def get_method_raw(self, k): @@ -348,7 +348,7 @@ class FlowListBox(urwid.ListBox): if key == "A": for f in self.master.view: if f.intercepted: - f.resume() + f.resume(self.master) signals.flowlist_change.send(self) elif key == "z": self.master.view.clear() diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index e2b24fab9..ecb070d82 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -510,8 +510,10 @@ class FlowView(tabs.Tabs): self.flow.resume(self.master) signals.flow_change.send(self, flow = self.flow) elif key == "A": - self.master.accept_all() - signals.flow_change.send(self, flow = self.flow) + for f in self.view: + if f.intercepted: + f.resume(self.master) + signals.flow_change.send(self, flow=f) elif key == "d": if self.flow.killable: self.flow.kill(self.master) diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index f8850404a..834d6eadb 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -71,7 +71,7 @@ class ConsoleMaster(master.Master): def __init__(self, options, server): super().__init__(options, server) - self.view = view.View() + self.view = view.View() # type: view.View self.stream_path = None # This line is just for type hinting self.options = self.options # type: Options diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index fa0b702f6..f617bd085 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -11,6 +11,7 @@ import tornado.escape import tornado.web import tornado.websocket from mitmproxy import contentviews +from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import http from mitmproxy import io @@ -108,9 +109,23 @@ class RequestHandler(tornado.web.RequestHandler): @property def json(self): - if not self.request.headers.get("Content-Type").startswith("application/json"): - return None - return json.loads(self.request.body.decode()) + if not self.request.headers.get("Content-Type", "").startswith("application/json"): + raise APIError(400, "Invalid Content-Type, expected application/json.") + try: + return json.loads(self.request.body.decode()) + except Exception as e: + raise APIError(400, "Malformed JSON: {}".format(str(e))) + + @property + def filecontents(self): + """ + Accept either a multipart/form file upload or just take the plain request body. + + """ + if self.request.files: + return next(iter(self.request.files.values()))[0].body + else: + return self.request.body @property def view(self) -> mitmproxy.addons.view.View: @@ -124,11 +139,11 @@ class RequestHandler(tornado.web.RequestHandler): def flow(self) -> mitmproxy.flow.Flow: flow_id = str(self.path_kwargs["flow_id"]) # FIXME: Add a facility to addon.view to safely access the store - flow = self.view._store.get(flow_id) + flow = self.view.get_by_id(flow_id) if flow: return flow else: - raise APIError(400, "Flow not found.") + raise APIError(404, "Flow not found.") def write_error(self, status_code: int, **kwargs): if "exc_info" in kwargs and isinstance(kwargs["exc_info"][1], APIError): @@ -168,7 +183,7 @@ class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler): for conn in cls.connections: try: conn.write_message(message) - except Exception: + except Exception: # pragma: no cover logging.error("Error sending message", exc_info=True) @@ -196,10 +211,8 @@ class DumpFlows(RequestHandler): def post(self): self.view.clear() - - content = self.request.files.values()[0][0].body - bio = BytesIO(content) - self.master.load_flows(io.FlowReader(bio).stream()) + bio = BytesIO(self.filecontents) + self.master.load_flows(io.FlowReader(bio)) bio.close() @@ -211,7 +224,8 @@ class ClearAll(RequestHandler): class AcceptFlows(RequestHandler): def post(self): - self.master.accept_all(self.master) + for f in self.view: + f.resume(self.master) class AcceptFlow(RequestHandler): @@ -228,53 +242,59 @@ class FlowHandler(RequestHandler): def put(self, flow_id): flow = self.flow flow.backup() - for a, b in self.json.items(): - if a == "request" and hasattr(flow, "request"): - request = flow.request - for k, v in b.items(): - if k in ["method", "scheme", "host", "path", "http_version"]: - setattr(request, k, str(v)) - elif k == "port": - request.port = int(v) - elif k == "headers": - request.headers.clear() - for header in v: - request.headers.add(*header) - elif k == "content": - request.text = v - else: - print("Warning: Unknown update {}.{}: {}".format(a, k, v)) + try: + for a, b in self.json.items(): + if a == "request" and hasattr(flow, "request"): + request = flow.request + for k, v in b.items(): + if k in ["method", "scheme", "host", "path", "http_version"]: + setattr(request, k, str(v)) + elif k == "port": + request.port = int(v) + elif k == "headers": + request.headers.clear() + for header in v: + request.headers.add(*header) + elif k == "content": + request.text = v + else: + raise APIError(400, "Unknown update request.{}: {}".format(k, v)) - elif a == "response" and hasattr(flow, "response"): - response = flow.response - for k, v in b.items(): - if k == "msg": - response.msg = str(v) - elif k == "code": - response.status_code = int(v) - elif k == "http_version": - response.http_version = str(v) - elif k == "headers": - response.headers.clear() - for header in v: - response.headers.add(*header) - elif k == "content": - response.text = v - else: - print("Warning: Unknown update {}.{}: {}".format(a, k, v)) - else: - print("Warning: Unknown update {}: {}".format(a, b)) + elif a == "response" and hasattr(flow, "response"): + response = flow.response + for k, v in b.items(): + if k in ["msg", "http_version"]: + setattr(response, k, str(v)) + elif k == "code": + response.status_code = int(v) + elif k == "headers": + response.headers.clear() + for header in v: + response.headers.add(*header) + elif k == "content": + response.text = v + else: + raise APIError(400, "Unknown update response.{}: {}".format(k, v)) + else: + raise APIError(400, "Unknown update {}: {}".format(a, b)) + except APIError: + flow.revert() + raise self.view.update(flow) class DuplicateFlow(RequestHandler): def post(self, flow_id): - self.master.view.duplicate_flow(self.flow) + f = self.flow.copy() + self.view.add(f) + self.write(f.id) class RevertFlow(RequestHandler): def post(self, flow_id): - self.flow.revert() + if self.flow.modified(): + self.flow.revert() + self.view.update(self.flow) class ReplayFlow(RequestHandler): @@ -283,16 +303,17 @@ class ReplayFlow(RequestHandler): self.flow.response = None self.view.update(self.flow) - r = self.master.replay_request(self.flow) - if r: - raise APIError(400, r) + try: + self.master.replay_request(self.flow) + except exceptions.ReplayException as e: + raise APIError(400, str(e)) class FlowContent(RequestHandler): def post(self, flow_id, message): self.flow.backup() message = getattr(self.flow, message) - message.content = self.request.files.values()[0][0].body + message.content = self.filecontents self.view.update(self.flow) def get(self, flow_id, message): @@ -364,46 +385,16 @@ class Settings(RequestHandler): )) def put(self): - update = {} - for k, v in self.json.items(): - if k == "intercept": - self.master.options.intercept = v - update[k] = v - elif k == "showhost": - self.master.options.showhost = v - update[k] = v - elif k == "no_upstream_cert": - self.master.options.no_upstream_cert = v - update[k] = v - elif k == "rawtcp": - self.master.options.rawtcp = v - update[k] = v - elif k == "http2": - self.master.options.http2 = v - update[k] = v - elif k == "anticache": - self.master.options.anticache = v - update[k] = v - elif k == "anticomp": - self.master.options.anticomp = v - update[k] = v - elif k == "stickycookie": - self.master.options.stickycookie = v - update[k] = v - elif k == "stickyauth": - self.master.options.stickyauth = v - update[k] = v - elif k == "stream": - self.master.options.stream_large_bodies = v - update[k] = v - else: - print("Warning: Unknown setting {}: {}".format(k, v)) - - ClientConnection.broadcast( - resource="settings", - cmd="update", - data=update - ) + update = self.json + option_whitelist = { + "intercept", "showhost", "no_upstream_cert", + "rawtcp", "http2", "anticache", "anticomp", + "stickycookie", "stickyauth", "stream_large_bodies" + } + for k in update: + if k not in option_whitelist: + raise APIError(400, "Unknown setting {}".format(k)) + self.master.options.update(**update) class Application(tornado.web.Application): diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 453455f03..5457fb40a 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -45,6 +45,8 @@ class WebMaster(master.Master): self.events.sig_add.connect(self._sig_events_add) self.events.sig_refresh.connect(self._sig_events_refresh) + self.options.changed.connect(self._sig_options_update) + self.addons.add(*addons.default_addons()) self.addons.add(self.view, self.events, intercept.Intercept()) self.app = app.Application( @@ -101,6 +103,13 @@ class WebMaster(master.Master): cmd="reset" ) + def _sig_options_update(self, options, updated): + app.ClientConnection.broadcast( + resource="settings", + cmd="update", + data={k: getattr(options, k) for k in updated} + ) + def run(self): # pragma: no cover iol = tornado.ioloop.IOLoop.instance() diff --git a/test/mitmproxy/data/addonscripts/duplicate_flow.py b/test/mitmproxy/data/addonscripts/duplicate_flow.py deleted file mode 100644 index 02fb8dcec..000000000 --- a/test/mitmproxy/data/addonscripts/duplicate_flow.py +++ /dev/null @@ -1,6 +0,0 @@ -from mitmproxy import ctx - - -def request(flow): - f = ctx.master.state.duplicate_flow(flow) - ctx.master.replay_request(f, block=True) diff --git a/test/mitmproxy/test_web_app.py b/test/mitmproxy/test_web_app.py index 8fc3378a2..be1955282 100644 --- a/test/mitmproxy/test_web_app.py +++ b/test/mitmproxy/test_web_app.py @@ -1,15 +1,47 @@ -import tornado.testing +import json as _json +import mock +import tornado.testing +from mitmproxy import exceptions from mitmproxy import proxy +from mitmproxy.test import tflow from mitmproxy.tools.web import app from mitmproxy.tools.web import master as webmaster +from tornado import httpclient +from tornado import websocket + + +def json(resp: httpclient.HTTPResponse): + return _json.loads(resp.body.decode()) class TestApp(tornado.testing.AsyncHTTPTestCase): def get_app(self): o = webmaster.Options() m = webmaster.WebMaster(o, proxy.DummyServer()) - return app.Application(m, None, None) + f = tflow.tflow(resp=True) + f.id = "42" + m.view.add(f) + m.view.add(tflow.tflow(err=True)) + m.add_log("test log", "info") + self.master = m + self.view = m.view + self.events = m.events + webapp = app.Application(m, None) + webapp.settings["xsrf_cookies"] = False + return webapp + + def fetch(self, *args, **kwargs) -> httpclient.HTTPResponse: + # tornado disallows POST without content by default. + return super().fetch(*args, **kwargs, allow_nonstandard_methods=True) + + def put_json(self, url, data: dict) -> httpclient.HTTPResponse: + return self.fetch( + url, + method="PUT", + body=_json.dumps(data), + headers={"Content-Type": "application/json"}, + ) def test_index(self): assert self.fetch("/").code == 200 @@ -17,8 +49,217 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): def test_filter_help(self): assert self.fetch("/filter-help").code == 200 - def test_events(self): - assert self.fetch("/events").code == 200 - def test_flows(self): - assert self.fetch("/flows").code == 200 + resp = self.fetch("/flows") + assert resp.code == 200 + assert json(resp)[0]["request"]["contentHash"] + assert json(resp)[1]["error"] + + def test_flows_dump(self): + resp = self.fetch("/flows/dump") + assert b"address" in resp.body + + self.view.clear() + assert not len(self.view) + + assert self.fetch("/flows/dump", method="POST", body=resp.body).code == 200 + assert len(self.view) + + def test_clear(self): + events = self.events.data.copy() + flows = list(self.view) + + assert self.fetch("/clear", method="POST").code == 200 + + assert not len(self.view) + assert not len(self.events.data) + + # restore + for f in flows: + self.view.add(f) + self.events.data = events + + def test_accept(self): + for f in self.view: + f.reply.handle() + f.intercept(self.master) + + assert self.fetch( + "/flows/42/accept", method="POST").code == 200 + assert sum(f.intercepted for f in self.view) == 1 + assert self.fetch("/flows/accept", method="POST").code == 200 + assert all(not f.intercepted for f in self.view) + + def test_flow_delete(self): + f = self.view.get_by_id("42") + assert f + + f.reply.handle() + assert self.fetch("/flows/42", method="DELETE").code == 200 + + assert not self.view.get_by_id("42") + self.view.add(f) + + assert self.fetch("/flows/1234", method="DELETE").code == 404 + + def test_flow_update(self): + f = self.view.get_by_id("42") + assert f.request.method == "GET" + f.backup() + + upd = { + "request": { + "method": "PATCH", + "port": 123, + "headers": [("foo", "bar")], + "content": "req", + }, + "response": { + "msg": "Not Found", + "code": 404, + "headers": [("bar", "baz")], + "content": "resp", + } + } + assert self.put_json("/flows/42", upd).code == 200 + assert f.request.method == "PATCH" + assert f.request.port == 123 + assert f.request.headers["foo"] == "bar" + assert f.request.text == "req" + assert f.response.msg == "Not Found" + assert f.response.status_code == 404 + assert f.response.headers["bar"] == "baz" + assert f.response.text == "resp" + + f.revert() + + assert self.put_json("/flows/42", {"foo": 42}).code == 400 + assert self.put_json("/flows/42", {"request": {"foo": 42}}).code == 400 + assert self.put_json("/flows/42", {"response": {"foo": 42}}).code == 400 + assert self.fetch("/flows/42", method="PUT", body="{}").code == 400 + assert self.fetch( + "/flows/42", + method="PUT", + headers={"Content-Type": "application/json"}, + body="!!" + ).code == 400 + + def test_flow_duplicate(self): + resp = self.fetch("/flows/42/duplicate", method="POST") + assert resp.code == 200 + f = self.view.get_by_id(resp.body.decode()) + assert f + assert f.id != "42" + self.view.remove(f) + + def test_flow_revert(self): + f = self.view.get_by_id("42") + f.backup() + f.request.method = "PATCH" + self.fetch("/flows/42/revert", method="POST") + assert not f._backup + + def test_flow_replay(self): + with mock.patch("mitmproxy.master.Master.replay_request") as replay_request: + assert self.fetch("/flows/42/replay", method="POST").code == 200 + assert replay_request.called + replay_request.side_effect = exceptions.ReplayException( + "out of replays" + ) + assert self.fetch("/flows/42/replay", method="POST").code == 400 + + def test_flow_content(self): + f = self.view.get_by_id("42") + f.backup() + f.response.headers["Content-Encoding"] = "ran\x00dom" + f.response.headers["Content-Disposition"] = 'inline; filename="filename.jpg"' + + r = self.fetch("/flows/42/response/content") + assert r.body == b"message" + assert r.headers["Content-Encoding"] == "random" + assert r.headers["Content-Disposition"] == 'attachment; filename="filename.jpg"' + + del f.response.headers["Content-Disposition"] + f.request.path = "/foo/bar.jpg" + assert self.fetch( + "/flows/42/response/content" + ).headers["Content-Disposition"] == 'attachment; filename=bar.jpg' + + f.response.content = b"" + assert self.fetch("/flows/42/response/content").code == 400 + + f.revert() + + def test_update_flow_content(self): + assert self.fetch( + "/flows/42/request/content", + method="POST", + body="new" + ).code == 200 + f = self.view.get_by_id("42") + assert f.request.content == b"new" + assert f.modified() + f.revert() + + def test_update_flow_content_multipart(self): + body = ( + b'--somefancyboundary\r\n' + b'Content-Disposition: form-data; name="a"; filename="a.txt"\r\n' + b'\r\n' + b'such multipart. very wow.\r\n' + b'--somefancyboundary--\r\n' + ) + assert self.fetch( + "/flows/42/request/content", + method="POST", + headers={"Content-Type": 'multipart/form-data; boundary="somefancyboundary"'}, + body=body + ).code == 200 + f = self.view.get_by_id("42") + assert f.request.content == b"such multipart. very wow." + assert f.modified() + f.revert() + + def test_flow_content_view(self): + assert json(self.fetch("/flows/42/request/content/raw")) == { + "lines": [ + [["text", "content"]] + ], + "description": "Raw" + } + + def test_events(self): + resp = self.fetch("/events") + assert resp.code == 200 + assert json(resp)[0]["level"] == "info" + + def test_settings(self): + assert json(self.fetch("/settings"))["mode"] == "regular" + + def test_settings_update(self): + assert self.put_json("/settings", {"anticache": True}).code == 200 + assert self.put_json("/settings", {"wtf": True}).code == 400 + + def test_err(self): + with mock.patch("mitmproxy.tools.web.app.IndexHandler.get") as f: + f.side_effect = RuntimeError + assert self.fetch("/").code == 500 + + @tornado.testing.gen_test + def test_websocket(self): + ws_url = "ws://localhost:{}/updates".format(self.get_http_port()) + + ws_client = yield websocket.websocket_connect(ws_url) + self.master.options.anticomp = True + + response = yield ws_client.read_message() + assert _json.loads(response) == { + "resource": "settings", + "cmd": "update", + "data": {"anticomp": True}, + } + ws_client.close() + + # trigger on_close by opening a second connection. + ws_client2 = yield websocket.websocket_connect(ws_url) + ws_client2.close() diff --git a/web/src/js/components/Footer.jsx b/web/src/js/components/Footer.jsx index 96e7b7db4..1ae4ee739 100644 --- a/web/src/js/components/Footer.jsx +++ b/web/src/js/components/Footer.jsx @@ -7,7 +7,7 @@ Footer.propTypes = { } function Footer({ settings }) { - let {mode, intercept, showhost, no_upstream_cert, rawtcp, http2, anticache, anticomp, stickyauth, stickycookie, stream} = settings; + let {mode, intercept, showhost, no_upstream_cert, rawtcp, http2, anticache, anticomp, stickyauth, stickycookie, stream_large_bodies} = settings; return (
{mode && mode != "regular" && ( @@ -40,8 +40,8 @@ function Footer({ settings }) { {stickycookie && ( stickycookie: {stickycookie} )} - {stream && ( - stream: {formatSize(stream)} + {stream_large_bodies && ( + stream: {formatSize(stream_large_bodies)} )}
) diff --git a/web/src/js/components/Header/OptionMenu.jsx b/web/src/js/components/Header/OptionMenu.jsx index a11062f2f..186a9c6a1 100644 --- a/web/src/js/components/Header/OptionMenu.jsx +++ b/web/src/js/components/Header/OptionMenu.jsx @@ -49,11 +49,11 @@ function OptionMenu({ settings, updateSettings }) { txt={settings.stickycookie} onToggleChanged={txt => updateSettings({ stickycookie: !settings.stickycookie ? txt : null })} /> - updateSettings({ stream: !settings.stream ? txt : null })} + onToggleChanged={txt => updateSettings({ stream_large_bodies: !settings.stream_large_bodies ? txt : null })} />