From d8a78d9f52d74d7742b199233eb9f119fd035bc4 Mon Sep 17 00:00:00 2001 From: Clemens Date: Wed, 27 Jul 2016 09:45:46 +0200 Subject: [PATCH 01/18] add contentviews to settings --- mitmproxy/web/app.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index f8f85f3dd..812aa680a 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -16,9 +16,11 @@ from mitmproxy.flow import FlowWriter, FlowReader from mitmproxy import filt from mitmproxy import models +from mitmproxy import contentviews from netlib import version + def convert_flow_to_json_dict(flow): # type: (models.Flow) -> dict """ @@ -364,7 +366,8 @@ class Settings(RequestHandler): anticomp=self.master.options.anticomp, stickyauth=self.master.options.stickyauth, stickycookie=self.master.options.stickycookie, - stream= self.master.options.stream_large_bodies + stream= self.master.options.stream_large_bodies, + contentViews= map(lambda v : v.name, contentviews.views) ) )) From 2c3ddb0ecc3d57b099057ede73da5576655b0583 Mon Sep 17 00:00:00 2001 From: Clemens Date: Wed, 27 Jul 2016 10:21:06 +0200 Subject: [PATCH 02/18] add parameter to content requests --- mitmproxy/web/app.py | 1 + web/src/js/flow/utils.js | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index 812aa680a..5f756ed1d 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -314,6 +314,7 @@ class FlowContent(RequestHandler): def get(self, flow_id, message): message = getattr(self.flow, message) + contentview = self.get_argument("content_view", "raw", True) if not message.raw_content: raise APIError(400, "No content.") diff --git a/web/src/js/flow/utils.js b/web/src/js/flow/utils.js index d24f984ca..0232e7533 100644 --- a/web/src/js/flow/utils.js +++ b/web/src/js/flow/utils.js @@ -43,13 +43,13 @@ export var MessageUtils = { } return false; }, - getContentURL: function (flow, message) { + getContentURL: function (flow, message, view) { if (message === flow.request) { message = "request"; } else if (message === flow.response) { message = "response"; } - return "/flows/" + flow.id + "/" + message + "/content"; + return "/flows/" + flow.id + "/" + message + "/content" + (view ? "?content_view="+view : ""); }, }; From 3e6c284757ffbb42aef34281567cf1adedd21ae9 Mon Sep 17 00:00:00 2001 From: Clemens Date: Wed, 27 Jul 2016 17:27:01 +0200 Subject: [PATCH 03/18] add contentviews to client --- web/src/js/components/ContentView.jsx | 4 ++-- web/src/js/components/ContentView/ContentLoader.jsx | 5 +++-- web/src/js/components/ContentView/ContentViews.jsx | 12 +++++++++++- web/src/js/components/ContentView/ViewSelector.jsx | 11 ++++++++++- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/web/src/js/components/ContentView.jsx b/web/src/js/components/ContentView.jsx index 756625097..a93ce3950 100644 --- a/web/src/js/components/ContentView.jsx +++ b/web/src/js/components/ContentView.jsx @@ -33,10 +33,10 @@ function ContentView(props) { return } - const View = ContentViews[contentView] + const View = ContentViews[contentView] || ContentViews['ViewServer'] return (
- +
diff --git a/web/src/js/components/ContentView/ContentLoader.jsx b/web/src/js/components/ContentView/ContentLoader.jsx index ba6702ca8..9babb8f79 100644 --- a/web/src/js/components/ContentView/ContentLoader.jsx +++ b/web/src/js/components/ContentView/ContentLoader.jsx @@ -28,7 +28,8 @@ export default View => class extends React.Component { componentWillReceiveProps(nextProps) { if ( nextProps.message.content !== this.props.message.content || - nextProps.message.contentHash !== this.props.message.contentHash + nextProps.message.contentHash !== this.props.message.contentHash || + nextProps.contentView !== this.props.contentView ) { this.updateContent(nextProps) } @@ -52,7 +53,7 @@ export default View => class extends React.Component { return this.setState({request: undefined, content: ""}) } - let requestUrl = MessageUtils.getContentURL(props.flow, props.message) + let requestUrl = MessageUtils.getContentURL(props.flow, props.message, props.contentView) // We use XMLHttpRequest instead of fetch() because fetch() is not (yet) abortable. let request = new XMLHttpRequest(); diff --git a/web/src/js/components/ContentView/ContentViews.jsx b/web/src/js/components/ContentView/ContentViews.jsx index a1adebea5..732f9f5e8 100644 --- a/web/src/js/components/ContentView/ContentViews.jsx +++ b/web/src/js/components/ContentView/ContentViews.jsx @@ -57,4 +57,14 @@ function ViewAuto({ message, flow, readonly, onChange }) { return } -export { ViewImage, ViewRaw, ViewAuto, ViewJSON } +function ViewServer({contentView, content}){ + return
+ 
load from server this view: {contentView}
+ 
{content}
+
+ +} + +ViewServer = ContentLoader(ViewServer) + +export { ViewImage, ViewRaw, ViewAuto, ViewJSON, ViewServer } diff --git a/web/src/js/components/ContentView/ViewSelector.jsx b/web/src/js/components/ContentView/ViewSelector.jsx index 89b362314..c3e1e1059 100644 --- a/web/src/js/components/ContentView/ViewSelector.jsx +++ b/web/src/js/components/ContentView/ViewSelector.jsx @@ -24,7 +24,7 @@ ViewButton = connect(state => ({ ViewSelector.propTypes = { message: PropTypes.object.isRequired, } -export default function ViewSelector({ message }) { +function ViewSelector({ message, contentViews }) { let autoView = ContentViews.ViewAuto.findView(message) let autoViewName = (autoView.displayName || autoView.name) @@ -42,6 +42,15 @@ export default function ViewSelector({ message }) { {name.toLowerCase().replace('view', '')} )} + {contentViews.map(name => + {name.toLowerCase().replace('view', '')} + )} +
) } + +export default connect ( + state => ({ + contentViews: state.settings.contentViews + }))(ViewSelector) From ad5bebeda0f1da890d8ed94b6096515e03f44a25 Mon Sep 17 00:00:00 2001 From: Clemens Date: Thu, 28 Jul 2016 17:10:06 +0200 Subject: [PATCH 04/18] mitmproxy.contentviews added --- mitmproxy/web/app.py | 50 +++++++++++++++++-- web/src/css/app.less | 1 + web/src/css/contentview.less | 10 ++++ web/src/js/components/ContentView.jsx | 2 +- .../components/ContentView/ContentLoader.jsx | 2 +- .../components/ContentView/ContentViews.jsx | 42 +++++++--------- .../components/ContentView/ViewSelector.jsx | 2 +- web/src/js/flow/utils.js | 4 +- 8 files changed, 81 insertions(+), 32 deletions(-) create mode 100644 web/src/css/contentview.less diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index 5f756ed1d..6abd672da 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -14,6 +14,7 @@ import tornado.web from io import BytesIO from mitmproxy.flow import FlowWriter, FlowReader +from mitmproxy import exceptions from mitmproxy import filt from mitmproxy import models from mitmproxy import contentviews @@ -305,6 +306,36 @@ class ReplayFlow(RequestHandler): class FlowContent(RequestHandler): + def _get_content_view(self, message, viewmode): + + try: + content = message.content + if content != message.raw_content: + enc = "[decoded {}]".format( + message.headers.get("content-encoding") + ) + else: + enc = None + except ValueError: + content = message.raw_content + enc = "[cannot decode]" + try: + query = None + if isinstance(message, models.HTTPRequest): + query = message.query + description, lines = contentviews.get_content_view( + viewmode, content, headers=message.headers, query=query + ) + except exceptions.ContentViewException: + description, lines = contentviews.get_content_view( + contentviews.get("Raw"), content, headers=message.headers + ) + description = description.replace("Raw", "Couldn't parse: falling back to Raw") + + if enc: + description = " ".join([enc, description]) + + return description, lines def post(self, flow_id, message): self.flow.backup() @@ -314,7 +345,6 @@ class FlowContent(RequestHandler): def get(self, flow_id, message): message = getattr(self.flow, message) - contentview = self.get_argument("content_view", "raw", True) if not message.raw_content: raise APIError(400, "No content.") @@ -339,7 +369,21 @@ class FlowContent(RequestHandler): self.set_header("Content-Type", "application/text") self.set_header("X-Content-Type-Options", "nosniff") self.set_header("X-Frame-Options", "DENY") - self.write(message.raw_content) + + cv = self.get_argument("cv", None) + if cv: + self.set_header("Content-Encoding", "") + viewmode = next(v for v in contentviews.views if v.name == cv) + description, lines = self._get_content_view( + message, viewmode + ) + + self.write(dict( + lines=list(lines), + description=description + )) + else: + self.write(message.raw_content) class Events(RequestHandler): @@ -368,7 +412,7 @@ class Settings(RequestHandler): stickyauth=self.master.options.stickyauth, stickycookie=self.master.options.stickycookie, stream= self.master.options.stream_large_bodies, - contentViews= map(lambda v : v.name, contentviews.views) + contentViews= [v.name for v in contentviews.views] ) )) diff --git a/web/src/css/app.less b/web/src/css/app.less index 6f27f4474..353e412a4 100644 --- a/web/src/css/app.less +++ b/web/src/css/app.less @@ -18,3 +18,4 @@ html { @import (less) "eventlog.less"; @import (less) "footer.less"; @import (less) "codemirror.less"; +@import (less) "contentview.less"; diff --git a/web/src/css/contentview.less b/web/src/css/contentview.less new file mode 100644 index 000000000..327dd6895 --- /dev/null +++ b/web/src/css/contentview.less @@ -0,0 +1,10 @@ +.contentview { + .header { + font-weight: bold; + } + .highlight{ + color: pink; + } + .offset{ } + .text{ } +} diff --git a/web/src/js/components/ContentView.jsx b/web/src/js/components/ContentView.jsx index a93ce3950..de4ffd063 100644 --- a/web/src/js/components/ContentView.jsx +++ b/web/src/js/components/ContentView.jsx @@ -35,7 +35,7 @@ function ContentView(props) { const View = ContentViews[contentView] || ContentViews['ViewServer'] return ( -
+
diff --git a/web/src/js/components/ContentView/ContentLoader.jsx b/web/src/js/components/ContentView/ContentLoader.jsx index 9babb8f79..e7a6f379f 100644 --- a/web/src/js/components/ContentView/ContentLoader.jsx +++ b/web/src/js/components/ContentView/ContentLoader.jsx @@ -53,7 +53,7 @@ export default View => class extends React.Component { return this.setState({request: undefined, content: ""}) } - let requestUrl = MessageUtils.getContentURL(props.flow, props.message, props.contentView) + let requestUrl = MessageUtils.getContentURL(props.flow, props.message, (View.name == 'ViewServer' ? props.contentView : undefined)) // We use XMLHttpRequest instead of fetch() because fetch() is not (yet) abortable. let request = new XMLHttpRequest(); diff --git a/web/src/js/components/ContentView/ContentViews.jsx b/web/src/js/components/ContentView/ContentViews.jsx index 732f9f5e8..89e97267b 100644 --- a/web/src/js/components/ContentView/ContentViews.jsx +++ b/web/src/js/components/ContentView/ContentViews.jsx @@ -28,26 +28,8 @@ function ViewRaw({ content, readonly, onChange }) { } ViewRaw = ContentLoader(ViewRaw) - -const isJSON = /^application\/json$/i -ViewJSON.matches = msg => isJSON.test(MessageUtils.getContentType(msg)) -ViewJSON.propTypes = { - content: React.PropTypes.string.isRequired, -} -function ViewJSON({ content }) { - let json = content - try { - json = JSON.stringify(JSON.parse(content), null, 2); - } catch (e) { - // @noop - } - return 
{json}
-} -ViewJSON = ContentLoader(ViewJSON) - - ViewAuto.matches = () => false -ViewAuto.findView = msg => [ViewImage, ViewJSON, ViewRaw].find(v => v.matches(msg)) || ViewRaw +ViewAuto.findView = msg => [ViewImage, ViewRaw].find(v => v.matches(msg)) || ViewRaw ViewAuto.propTypes = { message: React.PropTypes.object.isRequired, flow: React.PropTypes.object.isRequired, @@ -57,14 +39,26 @@ function ViewAuto({ message, flow, readonly, onChange }) { return } -function ViewServer({contentView, content}){ +function ViewServer({content, contentView}){ + let data = JSON.parse(content) return
- 
load from server this view: {contentView}
- 
{content}
+ {contentView != data.description && +
{data.description}
+ } + 
+                {data.lines.map((line, i) =>
+                    

+                        {line.map((tuple, j) =>
+                            
+                                {tuple[1]}
+                            
+                        )}
+                    

+                )}
+
- } ViewServer = ContentLoader(ViewServer) -export { ViewImage, ViewRaw, ViewAuto, ViewJSON, ViewServer } +export { ViewImage, ViewRaw, ViewAuto, ViewServer } diff --git a/web/src/js/components/ContentView/ViewSelector.jsx b/web/src/js/components/ContentView/ViewSelector.jsx index c3e1e1059..423cc157d 100644 --- a/web/src/js/components/ContentView/ViewSelector.jsx +++ b/web/src/js/components/ContentView/ViewSelector.jsx @@ -38,7 +38,7 @@ function ViewSelector({ message, contentViews }) { auto: {autoViewName} {Object.keys(ContentViews).map(name => - name !== "ViewAuto" && + name !== "ViewAuto" && name !== "ViewServer" && {name.toLowerCase().replace('view', '')} )} diff --git a/web/src/js/flow/utils.js b/web/src/js/flow/utils.js index 0232e7533..b8435aa01 100644 --- a/web/src/js/flow/utils.js +++ b/web/src/js/flow/utils.js @@ -49,8 +49,8 @@ export var MessageUtils = { } else if (message === flow.response) { message = "response"; } - return "/flows/" + flow.id + "/" + message + "/content" + (view ? "?content_view="+view : ""); - }, + return `/flows/${flow.id}/${message}/content` + (view ? `?cv=${view}` : ''); + } }; export var RequestUtils = _.extend(MessageUtils, { From 2807329fb2ce576ae529afd991e3da3d5f81d637 Mon Sep 17 00:00:00 2001 From: Clemens Date: Thu, 28 Jul 2016 17:28:05 +0200 Subject: [PATCH 05/18] add image to mitmproxy.contentview.image --- web/src/js/components/ContentView/ContentViews.jsx | 10 +++++++++- web/src/js/components/ContentView/ViewSelector.jsx | 4 +--- web/src/js/ducks/ui/flow.js | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/web/src/js/components/ContentView/ContentViews.jsx b/web/src/js/components/ContentView/ContentViews.jsx index 89e97267b..a1bee54ef 100644 --- a/web/src/js/components/ContentView/ContentViews.jsx +++ b/web/src/js/components/ContentView/ContentViews.jsx @@ -39,8 +39,11 @@ function ViewAuto({ message, flow, readonly, onChange }) { return } -function ViewServer({content, contentView}){ + +function ViewServer({content, contentView, message, flow}){ let data = JSON.parse(content) + let showImage = isImage.test(MessageUtils.getContentType(message)) + return
{contentView != data.description &&
{data.description}
@@ -56,6 +59,11 @@ function ViewServer({content, contentView}){
)} + {showImage && +
+ preview +
+ }
} diff --git a/web/src/js/components/ContentView/ViewSelector.jsx b/web/src/js/components/ContentView/ViewSelector.jsx index 423cc157d..e031b51f5 100644 --- a/web/src/js/components/ContentView/ViewSelector.jsx +++ b/web/src/js/components/ContentView/ViewSelector.jsx @@ -35,10 +35,8 @@ function ViewSelector({ message, contentViews }) { return (
- auto: {autoViewName} - {Object.keys(ContentViews).map(name => - name !== "ViewAuto" && name !== "ViewServer" && + name === "ViewRaw" && {name.toLowerCase().replace('view', '')} )} diff --git a/web/src/js/ducks/ui/flow.js b/web/src/js/ducks/ui/flow.js index c94356765..549efb1d0 100644 --- a/web/src/js/ducks/ui/flow.js +++ b/web/src/js/ducks/ui/flow.js @@ -14,7 +14,7 @@ export const SET_CONTENT_VIEW = 'UI_FLOWVIEW_SET_CONTENT_VIEW', const defaultState = { displayLarge: false, modifiedFlow: false, - contentView: 'ViewAuto', + contentView: 'Auto', tab: 'request', } From a56c2ca731ff9c4a22438553dea0e4ecf7a51f1d Mon Sep 17 00:00:00 2001 From: Clemens Date: Fri, 29 Jul 2016 09:45:15 +0200 Subject: [PATCH 06/18] combine clientside and serverside contentviews --- mitmproxy/web/app.py | 105 +++++++++++------- web/src/css/contentview.less | 12 +- .../components/ContentView/ContentViews.jsx | 32 ++---- .../components/ContentView/ViewSelector.jsx | 24 ++-- web/src/js/ducks/ui/flow.js | 4 + web/src/js/flow/utils.js | 2 +- 6 files changed, 93 insertions(+), 86 deletions(-) diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index 6abd672da..eabdb1470 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -306,36 +306,6 @@ class ReplayFlow(RequestHandler): class FlowContent(RequestHandler): - def _get_content_view(self, message, viewmode): - - try: - content = message.content - if content != message.raw_content: - enc = "[decoded {}]".format( - message.headers.get("content-encoding") - ) - else: - enc = None - except ValueError: - content = message.raw_content - enc = "[cannot decode]" - try: - query = None - if isinstance(message, models.HTTPRequest): - query = message.query - description, lines = contentviews.get_content_view( - viewmode, content, headers=message.headers, query=query - ) - except exceptions.ContentViewException: - description, lines = contentviews.get_content_view( - contentviews.get("Raw"), content, headers=message.headers - ) - description = description.replace("Raw", "Couldn't parse: falling back to Raw") - - if enc: - description = " ".join([enc, description]) - - return description, lines def post(self, flow_id, message): self.flow.backup() @@ -369,21 +339,69 @@ class FlowContent(RequestHandler): self.set_header("Content-Type", "application/text") self.set_header("X-Content-Type-Options", "nosniff") self.set_header("X-Frame-Options", "DENY") + self.write(message.raw_content) - cv = self.get_argument("cv", None) - if cv: - self.set_header("Content-Encoding", "") - viewmode = next(v for v in contentviews.views if v.name == cv) - description, lines = self._get_content_view( - message, viewmode +class FlowContentView(RequestHandler): + def _get_content_view(self, message, viewmode): + + try: + content = message.content + if content != message.raw_content: + enc = "[decoded {}]".format( + message.headers.get("content-encoding") + ) + else: + enc = None + except ValueError: + content = message.raw_content + enc = "[cannot decode]" + try: + query = None + if isinstance(message, models.HTTPRequest): + query = message.query + description, lines = contentviews.get_content_view( + viewmode, content, headers=message.headers, query=query ) + except exceptions.ContentViewException: + description, lines = contentviews.get_content_view( + contentviews.get("Raw"), content, headers=message.headers + ) + description = description.replace("Raw", "Couldn't parse: falling back to Raw") - self.write(dict( - lines=list(lines), - description=description - )) - else: - self.write(message.raw_content) + if enc: + description = " ".join([enc, description]) + + return description, lines + + def get(self, flow_id, message, content_view): + message = getattr(self.flow, message) + + original_cd = message.headers.get("Content-Disposition", None) + filename = None + if original_cd: + filename = re.search("filename=([\w\" \.\-\(\)]+)", original_cd) + if filename: + filename = filename.group(1) + if not filename: + filename = self.flow.request.path.split("?")[0].split("/")[-1] + + filename = re.sub(r"[^\w\" \.\-\(\)]", "", filename) + cd = "attachment; filename={}".format(filename) + self.set_header("Content-Disposition", cd) + self.set_header("Content-Type", "application/json") + self.set_header("X-Content-Type-Options", "nosniff") + self.set_header("X-Frame-Options", "DENY") + + self.set_header("Content-Encoding", "") + + description, lines = self._get_content_view( + message, contentviews.get(content_view.replace('_', ' ')) + ) + + self.write(dict( + lines=list(lines), + description=description + )) class Events(RequestHandler): @@ -412,7 +430,7 @@ class Settings(RequestHandler): stickyauth=self.master.options.stickyauth, stickycookie=self.master.options.stickycookie, stream= self.master.options.stream_large_bodies, - contentViews= [v.name for v in contentviews.views] + contentViews= [v.name.replace(' ', '_') for v in contentviews.views] ) )) @@ -477,6 +495,7 @@ class Application(tornado.web.Application): (r"/flows/(?P[0-9a-f\-]+)/replay", ReplayFlow), (r"/flows/(?P[0-9a-f\-]+)/revert", RevertFlow), (r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/content", FlowContent), + (r"/flows/(?P[0-9a-f\-]+)/(?Prequest|response)/content/(?P[0-9a-zA-Z\-\_]+)", FlowContentView), (r"/settings", Settings), (r"/clear", ClearAll), ] diff --git a/web/src/css/contentview.less b/web/src/css/contentview.less index 327dd6895..becac9a21 100644 --- a/web/src/css/contentview.less +++ b/web/src/css/contentview.less @@ -1,10 +1,14 @@ .contentview { .header { - font-weight: bold; + font-weight: bold; } .highlight{ - color: pink; + font-weight: bold; + } + .offset{ + color: blue + } + .text{ + } - .offset{ } - .text{ } } diff --git a/web/src/js/components/ContentView/ContentViews.jsx b/web/src/js/components/ContentView/ContentViews.jsx index a1bee54ef..3b2af0a90 100644 --- a/web/src/js/components/ContentView/ContentViews.jsx +++ b/web/src/js/components/ContentView/ContentViews.jsx @@ -18,31 +18,19 @@ function ViewImage({ flow, message }) { ) } - -ViewRaw.matches = () => true -ViewRaw.propTypes = { +Edit.propTypes = { content: React.PropTypes.string.isRequired, } -function ViewRaw({ content, readonly, onChange }) { - return readonly ? 
{content}
: -} -ViewRaw = ContentLoader(ViewRaw) -ViewAuto.matches = () => false -ViewAuto.findView = msg => [ViewImage, ViewRaw].find(v => v.matches(msg)) || ViewRaw -ViewAuto.propTypes = { - message: React.PropTypes.object.isRequired, - flow: React.PropTypes.object.isRequired, -} -function ViewAuto({ message, flow, readonly, onChange }) { - const View = ViewAuto.findView(message) - return +function Edit({ content, onChange }) { + return } +Edit = ContentLoader(Edit) -function ViewServer({content, contentView, message, flow}){ +function ViewServer(props){ + const {content, contentView, message} = props let data = JSON.parse(content) - let showImage = isImage.test(MessageUtils.getContentType(message)) return
{contentView != data.description && @@ -59,14 +47,12 @@ function ViewServer({content, contentView, message, flow}){
)} - {showImage && -
- preview -
+ {ViewImage.matches(message) && + }
} ViewServer = ContentLoader(ViewServer) -export { ViewImage, ViewRaw, ViewAuto, ViewServer } +export { Edit, ViewServer, ViewImage } diff --git a/web/src/js/components/ContentView/ViewSelector.jsx b/web/src/js/components/ContentView/ViewSelector.jsx index e031b51f5..c5670328b 100644 --- a/web/src/js/components/ContentView/ViewSelector.jsx +++ b/web/src/js/components/ContentView/ViewSelector.jsx @@ -24,25 +24,18 @@ ViewButton = connect(state => ({ ViewSelector.propTypes = { message: PropTypes.object.isRequired, } -function ViewSelector({ message, contentViews }) { - - let autoView = ContentViews.ViewAuto.findView(message) - let autoViewName = (autoView.displayName || autoView.name) - .toLowerCase() - .replace('view', '') - .replace(/ContentLoader\((.+)\)/,"$1") - +function ViewSelector({contentViews, isEdit }) { + let edit = ContentViews.Edit.displayName return (
- {Object.keys(ContentViews).map(name => - name === "ViewRaw" && - {name.toLowerCase().replace('view', '')} + {contentViews.map(name => + {name.toLowerCase().replace('_', ' ')} )} - {contentViews.map(name => - {name.toLowerCase().replace('view', '')} - )} + {isEdit && + {edit.toLowerCase()} + }
) @@ -50,5 +43,6 @@ function ViewSelector({ message, contentViews }) { export default connect ( state => ({ - contentViews: state.settings.contentViews + contentViews: state.settings.contentViews, + isEdit: !!state.ui.flow.modifiedFlow, }))(ViewSelector) diff --git a/web/src/js/ducks/ui/flow.js b/web/src/js/ducks/ui/flow.js index 549efb1d0..d9811a33c 100644 --- a/web/src/js/ducks/ui/flow.js +++ b/web/src/js/ducks/ui/flow.js @@ -19,12 +19,14 @@ const defaultState = { } export default function reducer(state = defaultState, action) { + let wasInEditMode = !!(state.modifiedFlow) switch (action.type) { case START_EDIT: return { ...state, modifiedFlow: action.flow, + contentView: 'Edit' } case UPDATE_EDIT: @@ -38,6 +40,7 @@ export default function reducer(state = defaultState, action) { ...state, modifiedFlow: false, displayLarge: false, + contentView: (wasInEditMode ? 'Auto' : state.contentView) } case flowsActions.UPDATE: @@ -49,6 +52,7 @@ export default function reducer(state = defaultState, action) { ...state, modifiedFlow: false, displayLarge: false, + contentView: (wasInEditMode ? 'Auto' : state.contentView) } } else { return state diff --git a/web/src/js/flow/utils.js b/web/src/js/flow/utils.js index b8435aa01..cd174069d 100644 --- a/web/src/js/flow/utils.js +++ b/web/src/js/flow/utils.js @@ -49,7 +49,7 @@ export var MessageUtils = { } else if (message === flow.response) { message = "response"; } - return `/flows/${flow.id}/${message}/content` + (view ? `?cv=${view}` : ''); + return `/flows/${flow.id}/${message}/content` + (view ? `/${view}` : ''); } }; From cb7da1ee44b19c15036459c2c9fdb2a34a3565f2 Mon Sep 17 00:00:00 2001 From: Clemens Date: Fri, 29 Jul 2016 15:37:41 +0200 Subject: [PATCH 07/18] moved getlines to contentviews --- mitmproxy/contentviews.py | 32 ++++++++++++++++++++++++++++++++ mitmproxy/web/app.py | 33 +-------------------------------- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index e155bc01e..88d979e44 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -19,6 +19,7 @@ import json import logging import subprocess import sys +import math from typing import Mapping # noqa @@ -30,6 +31,7 @@ from PIL import ExifTags from PIL import Image from six import BytesIO +from mitmproxy import models from mitmproxy import exceptions from mitmproxy.contrib import jsbeautifier from mitmproxy.contrib.wbxml import ASCommandResponse @@ -617,6 +619,36 @@ def safe_to_print(lines, encoding="utf8"): yield clean_line +def get_content_view_with_message_encoding(message, viewmode): + try: + content = message.content + if content != message.raw_content: + enc = "[decoded {}]".format( + message.headers.get("content-encoding") + ) + else: + enc = None + except ValueError: + content = message.raw_content + enc = "[cannot decode]" + try: + query = None + if isinstance(message, models.HTTPRequest): + query = message.query + description, lines = get_content_view( + viewmode, content, headers=message.headers, query=query + ) + except exceptions.ContentViewException: + description, lines = get_content_view( + get("Raw"), content, headers=message.headers + ) + description = description.replace("Raw", "Couldn't parse: falling back to Raw") + + if enc: + description = " ".join([enc, description]) + + return description, lines + def get_content_view(viewmode, data, **metadata): """ Args: diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index eabdb1470..5c95c06f6 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -14,7 +14,6 @@ import tornado.web from io import BytesIO from mitmproxy.flow import FlowWriter, FlowReader -from mitmproxy import exceptions from mitmproxy import filt from mitmproxy import models from mitmproxy import contentviews @@ -342,36 +341,6 @@ class FlowContent(RequestHandler): self.write(message.raw_content) class FlowContentView(RequestHandler): - def _get_content_view(self, message, viewmode): - - try: - content = message.content - if content != message.raw_content: - enc = "[decoded {}]".format( - message.headers.get("content-encoding") - ) - else: - enc = None - except ValueError: - content = message.raw_content - enc = "[cannot decode]" - try: - query = None - if isinstance(message, models.HTTPRequest): - query = message.query - description, lines = contentviews.get_content_view( - viewmode, content, headers=message.headers, query=query - ) - except exceptions.ContentViewException: - description, lines = contentviews.get_content_view( - contentviews.get("Raw"), content, headers=message.headers - ) - description = description.replace("Raw", "Couldn't parse: falling back to Raw") - - if enc: - description = " ".join([enc, description]) - - return description, lines def get(self, flow_id, message, content_view): message = getattr(self.flow, message) @@ -394,7 +363,7 @@ class FlowContentView(RequestHandler): self.set_header("Content-Encoding", "") - description, lines = self._get_content_view( + description, lines = contentviews.get_content_view_with_message_encoding( message, contentviews.get(content_view.replace('_', ' ')) ) From bcc496527ebf5faf94025ec7c28992a1ac368140 Mon Sep 17 00:00:00 2001 From: Clemens Date: Mon, 1 Aug 2016 15:21:00 +0200 Subject: [PATCH 08/18] replace get_content_view in dumper and flowview --- mitmproxy/builtins/dumper.py | 18 ++++-------------- mitmproxy/console/flowview.py | 32 +++----------------------------- 2 files changed, 7 insertions(+), 43 deletions(-) diff --git a/mitmproxy/builtins/dumper.py b/mitmproxy/builtins/dumper.py index 59f9349d7..b1367e12e 100644 --- a/mitmproxy/builtins/dumper.py +++ b/mitmproxy/builtins/dumper.py @@ -73,20 +73,10 @@ class Dumper(object): elif content: self.echo("") - try: - _, lines = contentviews.get_content_view( - contentviews.get("Auto"), - content, - headers=getattr(message, "headers", None) - ) - except exceptions.ContentViewException: - s = "Content viewer failed: \n" + traceback.format_exc() - ctx.log.debug(s) - _, lines = contentviews.get_content_view( - contentviews.get("Raw"), - content, - headers=getattr(message, "headers", None) - ) + _, lines = contentviews.get_content_view_with_message_encoding( + message, + contentviews.get("Auto") + ) styles = dict( highlight=dict(bold=True), diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index c354563f0..c1001a5a4 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -207,35 +207,9 @@ class FlowView(tabs.Tabs): def _get_content_view(self, message, viewmode, max_lines, _): - try: - content = message.content - if content != message.raw_content: - enc = "[decoded {}]".format( - message.headers.get("content-encoding") - ) - else: - enc = None - except ValueError: - content = message.raw_content - enc = "[cannot decode]" - try: - query = None - if isinstance(message, models.HTTPRequest): - query = message.query - description, lines = contentviews.get_content_view( - viewmode, content, headers=message.headers, query=query - ) - except exceptions.ContentViewException: - s = "Content viewer failed: \n" + traceback.format_exc() - signals.add_log(s, "error") - description, lines = contentviews.get_content_view( - contentviews.get("Raw"), content, headers=message.headers - ) - description = description.replace("Raw", "Couldn't parse: falling back to Raw") - - if enc: - description = " ".join([enc, description]) - + description, lines = contentviews.get_content_view_with_message_encoding( + message, viewmode + ) # Give hint that you have to tab for the response. if description == "No content" and isinstance(message, models.HTTPRequest): description = "No request content (press tab to view response)" From 34fe391afbe18f89d774137f82620024f697ab6a Mon Sep 17 00:00:00 2001 From: Clemens Date: Wed, 3 Aug 2016 12:08:10 +0200 Subject: [PATCH 09/18] add view all button, add dropdown for contentviews --- web/src/css/flowdetail.less | 1 + web/src/js/components/ContentView.jsx | 10 +- .../components/ContentView/ContentViews.jsx | 75 ++++++++++----- .../ContentView/ShowFullContentButton.jsx | 29 ++++++ .../components/ContentView/ViewSelector.jsx | 92 +++++++++++++------ web/src/js/components/common/Button.jsx | 5 +- web/src/js/ducks/ui/flow.js | 65 +++++++++++-- 7 files changed, 213 insertions(+), 64 deletions(-) create mode 100644 web/src/js/components/ContentView/ShowFullContentButton.jsx diff --git a/web/src/css/flowdetail.less b/web/src/css/flowdetail.less index d450bca50..7e65528fc 100644 --- a/web/src/css/flowdetail.less +++ b/web/src/css/flowdetail.less @@ -65,6 +65,7 @@ } .view-options { + margin-bottom: 10px; margin-top: 10px; } diff --git a/web/src/js/components/ContentView.jsx b/web/src/js/components/ContentView.jsx index de4ffd063..9ec283cac 100644 --- a/web/src/js/components/ContentView.jsx +++ b/web/src/js/components/ContentView.jsx @@ -5,6 +5,7 @@ import * as MetaViews from './ContentView/MetaViews' import ViewSelector from './ContentView/ViewSelector' import UploadContentButton from './ContentView/UploadContentButton' import DownloadContentButton from './ContentView/DownloadContentButton' +import ShowFullContentButton from './ContentView/ShowFullContentButton' import { setContentView, displayLarge, updateEdit } from '../ducks/ui/flow' @@ -19,7 +20,7 @@ ContentView.propTypes = { ContentView.isContentTooLarge = msg => msg.contentLength > 1024 * 1024 * (ContentViews.ViewImage.matches(msg) ? 10 : 0.2) function ContentView(props) { - const { flow, message, contentView, isDisplayLarge, displayLarge, uploadContent, onContentChange, readonly } = props + const { flow, message, contentView, isDisplayLarge, displayLarge, uploadContent, onContentChange, readonly, contentViewDescription } = props if (message.contentLength === 0 && readonly) { return @@ -37,13 +38,15 @@ function ContentView(props) { return (
- -
+ +
    +   + {contentViewDescription}
) @@ -53,6 +56,7 @@ export default connect( state => ({ contentView: state.ui.flow.contentView, isDisplayLarge: state.ui.flow.displayLarge, + contentViewDescription: state.ui.flow.viewDescription }), { displayLarge, diff --git a/web/src/js/components/ContentView/ContentViews.jsx b/web/src/js/components/ContentView/ContentViews.jsx index 3b2af0a90..43aece468 100644 --- a/web/src/js/components/ContentView/ContentViews.jsx +++ b/web/src/js/components/ContentView/ContentViews.jsx @@ -1,4 +1,6 @@ -import React, { PropTypes } from 'react' +import React, { PropTypes, Component } from 'react' +import { connect } from 'react-redux' +import { setContentViewDescription, setShowFullContent } from '../../ducks/ui/flow' import ContentLoader from './ContentLoader' import { MessageUtils } from '../../flow/utils' import CodeEditor from './CodeEditor' @@ -27,32 +29,63 @@ function Edit({ content, onChange }) { } Edit = ContentLoader(Edit) +class ViewServer extends Component { + constructor(props){ + super(props) + this.maxLines = 80 + } -function ViewServer(props){ - const {content, contentView, message} = props - let data = JSON.parse(content) + componentWillMount(){ + this.setContentView(this.props) + } + componentWillReceiveProps(nextProps){ + this.setContentView(nextProps) + } + setContentView(props){ + try { + this.data = JSON.parse(props.content) + }catch(err) { + this.data= {lines: [], description: err.message} + } - return
- {contentView != data.description && -
{data.description}
- } - 
-                {data.lines.map((line, i) =>
-                    

-                        {line.map((tuple, j) =>
-                            
-                                {tuple[1]}
-                            
-                        )}
-                    

-                )}
-
+ props.setContentViewDescription(props.contentView != this.data.description ? this.data.description : '') + + let isFullContentShown = this.data.lines.length < this.maxLines + if (isFullContentShown) props.setShowFullContent(true) + } + render() { + const {content, contentView, message} = this.props + + let lines = this.props.showFullContent ? this.data.lines : this.data.lines.slice(0, this.maxLines) + + return
+ 
+                    {lines.map((line, i) =>
+                        

+                            {line.map((tuple, j) =>
+                                
+                                    {tuple[1]}
+                                
+                            )}
+                        

+                    )}
+
{ViewImage.matches(message) && - + }
+ } + } -ViewServer = ContentLoader(ViewServer) +ViewServer = connect( + state => ({ + showFullContent: state.ui.flow.showFullContent + }), + { + setContentViewDescription, + setShowFullContent + } +)(ContentLoader(ViewServer)) export { Edit, ViewServer, ViewImage } diff --git a/web/src/js/components/ContentView/ShowFullContentButton.jsx b/web/src/js/components/ContentView/ShowFullContentButton.jsx new file mode 100644 index 000000000..a0217d32b --- /dev/null +++ b/web/src/js/components/ContentView/ShowFullContentButton.jsx @@ -0,0 +1,29 @@ +import React, { Component, PropTypes } from 'react' +import { connect } from 'react-redux' +import { render } from 'react-dom'; +import Button from '../common/Button'; +import { setShowFullContent } from '../../ducks/ui/flow' + + + +ShowFullContentButton.propTypes = { + setShowFullContent: PropTypes.func.isRequired, + showFullContent: PropTypes.bool.isRequired +} + +function ShowFullContentButton ( {setShowFullContent, showFullContent} ){ + + return ( + !showFullContent && +
  • + setContentView(name)}> + {children} + +
    • ) } -ViewButton = connect(state => ({ - activeView: state.ui.flow.contentView -}), { - setContentView -})(ViewButton) -ViewSelector.propTypes = { - message: PropTypes.object.isRequired, -} -function ViewSelector({contentViews, isEdit }) { - let edit = ContentViews.Edit.displayName - return ( -
    +/*ViewSelector.propTypes = { + contentViews: PropTypes.array.isRequired, + activeView: PropTypes.string.isRequired, + isEdit: PropTypes.bool.isRequired, + isContentViewSelectorOpen: PropTypes.bool.isRequired, + setContentViewSelectorOpen: PropTypes.func.isRequired +}*/ - {contentViews.map(name => - {name.toLowerCase().replace('_', ' ')} - )} - {isEdit && - {edit.toLowerCase()} - } +class ViewSelector extends Component { + constructor(props, context) { + super(props, context) + this.close = this.close.bind(this) + } + close() { + this.props.setContentViewSelectorOpen(false) + document.removeEventListener('click', this.close) + } -
    - ) + onDropdown(e){ + e.preventDefault() + this.props.setContentViewSelectorOpen(!this.props.isContentViewSelectorOpen) + document.addEventListener('click', this.close) + } + + render() { + const {contentViews, activeView, isEdit, isContentViewSelectorOpen, setContentViewSelectorOpen, setContentView} = this.props + let edit = ContentViews.Edit.displayName + + return ( +
    + this.onDropdown(e) } + href="#"> + View: {activeView} + +
      + {contentViews.map(name => + + {name.toLowerCase().replace('_', ' ')} + + )} + {isEdit && + + {edit.toLowerCase()} + + } +
    +
    + ) + } } export default connect ( state => ({ contentViews: state.settings.contentViews, + activeView: state.ui.flow.contentView, isEdit: !!state.ui.flow.modifiedFlow, - }))(ViewSelector) + isContentViewSelectorOpen: state.ui.flow.isContentViewSelectorOpen + }), { + setContentView, + setContentViewSelectorOpen + } +)(ViewSelector) diff --git a/web/src/js/components/common/Button.jsx b/web/src/js/components/common/Button.jsx index cd01af227..0ac807826 100644 --- a/web/src/js/components/common/Button.jsx +++ b/web/src/js/components/common/Button.jsx @@ -1,4 +1,5 @@ import React, { PropTypes } from 'react' +import classnames from 'classnames' Button.propTypes = { onClick: PropTypes.func.isRequired, @@ -6,9 +7,9 @@ Button.propTypes = { icon: PropTypes.string } -export default function Button({ onClick, text, icon, disabled }) { +export default function Button({ onClick, text, icon, disabled, isXs }) { return ( -
    {icon && ( )} diff --git a/web/src/js/ducks/ui/flow.js b/web/src/js/ducks/ui/flow.js index d9811a33c..9c1d6dea0 100644 --- a/web/src/js/ducks/ui/flow.js +++ b/web/src/js/ducks/ui/flow.js @@ -3,16 +3,22 @@ import { getDiff } from "../../utils" import _ from 'lodash' -export const SET_CONTENT_VIEW = 'UI_FLOWVIEW_SET_CONTENT_VIEW', - DISPLAY_LARGE = 'UI_FLOWVIEW_DISPLAY_LARGE', - SET_TAB = "UI_FLOWVIEW_SET_TAB", - START_EDIT = 'UI_FLOWVIEW_START_EDIT', - UPDATE_EDIT = 'UI_FLOWVIEW_UPDATE_EDIT', - UPLOAD_CONTENT = 'UI_FLOWVIEW_UPLOAD_CONTENT' +export const SET_CONTENT_VIEW = 'UI_FLOWVIEW_SET_CONTENT_VIEW', + DISPLAY_LARGE = 'UI_FLOWVIEW_DISPLAY_LARGE', + SET_TAB = "UI_FLOWVIEW_SET_TAB", + START_EDIT = 'UI_FLOWVIEW_START_EDIT', + UPDATE_EDIT = 'UI_FLOWVIEW_UPDATE_EDIT', + UPLOAD_CONTENT = 'UI_FLOWVIEW_UPLOAD_CONTENT', + SET_SHOW_FULL_CONTENT = 'UI_SET_SHOW_FULL_CONTENT', + SET_CONTENT_VIEW_DESCRIPTION = "UI_SET_CONTENT_VIEW_DESCRIPTION", + SET_CONTENT_VIEW_SELECTOR = "UI_SET_CONTENT_VIEW_SELECTOR" const defaultState = { displayLarge: false, + contentViewDescription: '', + showFullContent: false, + isContentViewSelectorOpen: false, modifiedFlow: false, contentView: 'Auto', tab: 'request', @@ -26,7 +32,8 @@ export default function reducer(state = defaultState, action) { return { ...state, modifiedFlow: action.flow, - contentView: 'Edit' + contentView: 'Edit', + showFullContent: true } case UPDATE_EDIT: @@ -40,7 +47,9 @@ export default function reducer(state = defaultState, action) { ...state, modifiedFlow: false, displayLarge: false, - contentView: (wasInEditMode ? 'Auto' : state.contentView) + contentView: (wasInEditMode ? 'Auto' : state.contentView), + viewDescription: '', + showFullContent: false, } case flowsActions.UPDATE: @@ -52,24 +61,46 @@ export default function reducer(state = defaultState, action) { ...state, modifiedFlow: false, displayLarge: false, - contentView: (wasInEditMode ? 'Auto' : state.contentView) + contentView: (wasInEditMode ? 'Auto' : state.contentView), + viewDescription: '', + showFullContent: false } } else { return state } + case SET_CONTENT_VIEW_DESCRIPTION: + return { + ...state, + viewDescription: action.description + } + + case SET_SHOW_FULL_CONTENT: + return { + ...state, + showFullContent: action.show + } + + + case SET_CONTENT_VIEW_SELECTOR: + return { + ...state, + isContentViewSelectorOpen: action.contentViewSelector + } case SET_TAB: return { ...state, tab: action.tab, displayLarge: false, + showFullContent: false } case SET_CONTENT_VIEW: return { ...state, contentView: action.contentView, + showFullContent: action.contentView == 'Edit' } case DISPLAY_LARGE: @@ -102,6 +133,22 @@ export function updateEdit(update) { return { type: UPDATE_EDIT, update } } +export function setContentViewDescription(description) { + return { type: SET_CONTENT_VIEW_DESCRIPTION, description } +} + +export function setShowFullContent(show) { + return { type: SET_SHOW_FULL_CONTENT, show } +} + +export function setContentViewSelectorOpen(open){ + return {type: SET_CONTENT_VIEW_SELECTOR, contentViewSelector: open} +} + +export function updateEdit(update) { + return { type: UPDATE_EDIT, update } +} + export function stopEdit(flow, modifiedFlow) { let diff = getDiff(flow, modifiedFlow) return flowsActions.update(flow, diff) From e036bc9304c76b63169da11c6721745484d6da10 Mon Sep 17 00:00:00 2001 From: Clemens Date: Wed, 3 Aug 2016 12:17:19 +0200 Subject: [PATCH 10/18] integrate simplified contentviews --- mitmproxy/builtins/dumper.py | 26 ++---- mitmproxy/console/flowview.py | 13 +-- mitmproxy/contentviews.py | 116 ++++++++++++------------- netlib/http/__init__.py | 3 +- test/mitmproxy/builtins/test_dumper.py | 11 ++- test/mitmproxy/test_contentview.py | 100 ++++++++++++--------- 6 files changed, 137 insertions(+), 132 deletions(-) diff --git a/mitmproxy/builtins/dumper.py b/mitmproxy/builtins/dumper.py index b1367e12e..a98c9b465 100644 --- a/mitmproxy/builtins/dumper.py +++ b/mitmproxy/builtins/dumper.py @@ -63,20 +63,12 @@ class Dumper(object): ) self.echo(headers, ident=4) if self.flow_detail >= 3: - try: - content = message.content - except ValueError: - content = message.get_content(strict=False) - - if content is None: - self.echo("(content missing)", ident=4) - elif content: - self.echo("") - - _, lines = contentviews.get_content_view_with_message_encoding( - message, - contentviews.get("Auto") + _, lines, error = contentviews.get_message_content_view( + contentviews.get("Auto"), + message ) + if error: + ctx.log.debug(error) styles = dict( highlight=dict(bold=True), @@ -95,13 +87,13 @@ class Dumper(object): else: lines_to_echo = lines - lines_to_echo = list(lines_to_echo) - content = u"\r\n".join( u"".join(colorful(line)) for line in lines_to_echo ) + if content: + self.echo("") + self.echo(content) - self.echo(content) if next(lines, None): self.echo("(cut off)", ident=4, dim=True) @@ -251,4 +243,4 @@ class Dumper(object): server=repr(f.server_conn.address), direction=direction, )) - self._echo_message(message) + self._echo_message(message) \ No newline at end of file diff --git a/mitmproxy/console/flowview.py b/mitmproxy/console/flowview.py index c1001a5a4..323aefd27 100644 --- a/mitmproxy/console/flowview.py +++ b/mitmproxy/console/flowview.py @@ -80,7 +80,7 @@ def _mkhelp(): ("r", "replay request"), ("V", "revert changes to request"), ("v", "view body in external viewer"), - ("w", "save all flows matching current limit"), + ("w", "save all flows matching current view filter"), ("W", "save this flow"), ("x", "delete body"), ("z", "encode/decode a request/response"), @@ -206,10 +206,11 @@ class FlowView(tabs.Tabs): ) def _get_content_view(self, message, viewmode, max_lines, _): - - description, lines = contentviews.get_content_view_with_message_encoding( - message, viewmode + description, lines, error = contentviews.get_message_content_view( + viewmode, message ) + if error: + signals.add_log(error, "error") # Give hint that you have to tab for the response. if description == "No content" and isinstance(message, models.HTTPRequest): description = "No request content (press tab to view response)" @@ -687,6 +688,7 @@ class FlowView(tabs.Tabs): keys = ( ("gzip", "z"), ("deflate", "d"), + ("brotli", "b"), ), callback = self.encode_callback, args = (conn,) @@ -700,6 +702,7 @@ class FlowView(tabs.Tabs): encoding_map = { "z": "gzip", "d": "deflate", + "b": "brotli", } conn.encode(encoding_map[key]) - signals.flow_change.send(self, flow = self.flow) + signals.flow_change.send(self, flow = self.flow) \ No newline at end of file diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py index 88d979e44..807638dc6 100644 --- a/mitmproxy/contentviews.py +++ b/mitmproxy/contentviews.py @@ -14,31 +14,27 @@ requests, the query parameters are passed as the ``query`` keyword argument. """ from __future__ import absolute_import, print_function, division +import cssutils import datetime +import html2text +import jsbeautifier import json import logging -import subprocess -import sys -import math - -from typing import Mapping # noqa - -import html2text import lxml.etree import lxml.html import six +import subprocess +import traceback from PIL import ExifTags from PIL import Image -from six import BytesIO - -from mitmproxy import models from mitmproxy import exceptions -from mitmproxy.contrib import jsbeautifier from mitmproxy.contrib.wbxml import ASCommandResponse from netlib import http from netlib import multidict -from netlib.http import url from netlib import strutils +from netlib.http import url +from six import BytesIO +from typing import Mapping # noqa try: import pyamf @@ -46,17 +42,6 @@ try: except ImportError: # pragma no cover pyamf = None -try: - import cssutils -except ImportError: # pragma no cover - cssutils = None -else: - cssutils.log.setLevel(logging.CRITICAL) - - cssutils.ser.prefs.keepComments = True - cssutils.ser.prefs.omitLastSemicolon = False - cssutils.ser.prefs.indentClosingBrace = False - cssutils.ser.prefs.validOnly = False # Default view cutoff *in lines* VIEW_CUTOFF = 512 @@ -273,7 +258,7 @@ class ViewHTMLOutline(View): content_types = ["text/html"] def __call__(self, data, **metadata): - data = data.decode("utf-8") + data = data.decode("utf-8", "replace") h = html2text.HTML2Text(baseurl="") h.ignore_images = True h.body_width = 0 @@ -400,6 +385,7 @@ class ViewJavaScript(View): def __call__(self, data, **metadata): opts = jsbeautifier.default_options() opts.indent_size = 2 + data = data.decode("utf-8", "replace") res = jsbeautifier.beautify(data, opts) return "JavaScript", format_text(res) @@ -412,11 +398,14 @@ class ViewCSS(View): ] def __call__(self, data, **metadata): - if cssutils: - sheet = cssutils.parseString(data) - beautified = sheet.cssText - else: - beautified = data + cssutils.log.setLevel(logging.CRITICAL) + cssutils.ser.prefs.keepComments = True + cssutils.ser.prefs.omitLastSemicolon = False + cssutils.ser.prefs.indentClosingBrace = False + cssutils.ser.prefs.validOnly = False + + sheet = cssutils.parseString(data) + beautified = sheet.cssText return "CSS", format_text(beautified) @@ -619,35 +608,38 @@ def safe_to_print(lines, encoding="utf8"): yield clean_line -def get_content_view_with_message_encoding(message, viewmode): +def get_message_content_view(viewmode, message): + """ + Like get_content_view, but also handles message encoding. + """ try: content = message.content - if content != message.raw_content: + except ValueError: + content = message.raw_content + enc = "[cannot decode]" + else: + if isinstance(message, http.Message) and content != message.raw_content: enc = "[decoded {}]".format( message.headers.get("content-encoding") ) else: enc = None - except ValueError: - content = message.raw_content - enc = "[cannot decode]" - try: - query = None - if isinstance(message, models.HTTPRequest): - query = message.query - description, lines = get_content_view( - viewmode, content, headers=message.headers, query=query - ) - except exceptions.ContentViewException: - description, lines = get_content_view( - get("Raw"), content, headers=message.headers - ) - description = description.replace("Raw", "Couldn't parse: falling back to Raw") + + if content is None: + return "", iter([[("error", "content missing")]]), None + + query = message.query if isinstance(message, http.Request) else None + headers = message.headers if isinstance(message, http.Message) else None + + description, lines, error = get_content_view( + viewmode, content, headers=headers, query=query + ) if enc: - description = " ".join([enc, description]) + description = "{} {}".format(enc, description) + + return description, lines, error - return description, lines def get_content_view(viewmode, data, **metadata): """ @@ -656,24 +648,24 @@ def get_content_view(viewmode, data, **metadata): data, **metadata: arguments passed to View instance. Returns: - A (description, content generator) tuple. + A (description, content generator, error) tuple. + If the content view raised an exception generating the view, + the exception is returned in error and the flow is formatted in raw mode. In contrast to calling the views directly, text is always safe-to-print unicode. - - Raises: - ContentViewException, if the content view threw an error. """ try: ret = viewmode(data, **metadata) + if ret is None: + ret = "Couldn't parse: falling back to Raw", get("Raw")(data, **metadata)[1] + desc, content = ret + error = None # Third-party viewers can fail in unexpected ways... - except Exception as e: - six.reraise( - exceptions.ContentViewException, - exceptions.ContentViewException(str(e)), - sys.exc_info()[2] - ) - if not ret: + except Exception: desc = "Couldn't parse: falling back to Raw" _, content = get("Raw")(data, **metadata) - else: - desc, content = ret - return desc, safe_to_print(content) + error = "{} Content viewer failed: \n{}".format( + getattr(viewmode, "name"), + traceback.format_exc() + ) + + return desc, safe_to_print(content), error \ No newline at end of file diff --git a/netlib/http/__init__.py b/netlib/http/__init__.py index af95f4d09..fdf4ef8f9 100644 --- a/netlib/http/__init__.py +++ b/netlib/http/__init__.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, print_function, division from netlib.http.request import Request from netlib.http.response import Response +from netlib.http.message import Message from netlib.http.headers import Headers, parse_content_type from netlib.http.message import decoded from netlib.http import http1, http2, status_codes, multipart @@ -11,4 +12,4 @@ __all__ = [ "Headers", "parse_content_type", "decoded", "http1", "http2", "status_codes", "multipart", -] +] \ No newline at end of file diff --git a/test/mitmproxy/builtins/test_dumper.py b/test/mitmproxy/builtins/test_dumper.py index 6287fe861..2d551bab3 100644 --- a/test/mitmproxy/builtins/test_dumper.py +++ b/test/mitmproxy/builtins/test_dumper.py @@ -15,7 +15,7 @@ class TestDumper(mastertest.MasterTest): d = dumper.Dumper() sio = StringIO() - updated = set(["tfile", "flow_detail"]) + updated = {"tfile", "flow_detail"} d.configure(dump.Options(tfile = sio, flow_detail = 0), updated) d.response(tutils.tflow()) assert not sio.getvalue() @@ -66,10 +66,9 @@ class TestDumper(mastertest.MasterTest): class TestContentView(mastertest.MasterTest): - @mock.patch("mitmproxy.contentviews.get_content_view") - def test_contentview(self, get_content_view): - se = exceptions.ContentViewException(""), ("x", iter([])) - get_content_view.side_effect = se + @mock.patch("mitmproxy.contentviews.ViewAuto.__call__") + def test_contentview(self, view_auto): + view_auto.side_effect = exceptions.ContentViewException("") s = state.State() sio = StringIO() @@ -82,4 +81,4 @@ class TestContentView(mastertest.MasterTest): d = dumper.Dumper() m.addons.add(o, d) self.invoke(m, "response", tutils.tflow()) - assert "Content viewer failed" in m.event_log[0][1] + assert "Content viewer failed" in m.event_log[0][1] \ No newline at end of file diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py index aad53b372..8e2042fb3 100644 --- a/test/mitmproxy/test_contentview.py +++ b/test/mitmproxy/test_contentview.py @@ -1,3 +1,4 @@ +import mock from mitmproxy.exceptions import ContentViewException from netlib.http import Headers from netlib.http import url @@ -5,6 +6,7 @@ from netlib import multidict import mitmproxy.contentviews as cv from . import tutils +import netlib.tutils try: import pyamf @@ -78,6 +80,7 @@ class TestContentView: v = cv.ViewHTMLOutline() s = b"


    one

    " assert v(s) + assert v(b'\xfe') def test_view_json(self): cv.VIEW_CUTOFF = 100 @@ -106,9 +109,10 @@ class TestContentView: def test_view_javascript(self): v = cv.ViewJavaScript() - assert v("[1, 2, 3]") - assert v("[1, 2, 3") - assert v("function(a){[1, 2, 3]}") + assert v(b"[1, 2, 3]") + assert v(b"[1, 2, 3") + assert v(b"function(a){[1, 2, 3]}") + assert v(b"\xfe") # invalid utf-8 def test_view_css(self): v = cv.ViewCSS() @@ -178,43 +182,6 @@ Larry assert f[0] == "Query" assert [x for x in f[1]] == [[("header", "foo: "), ("text", "bar")]] - def test_get_content_view(self): - r = cv.get_content_view( - cv.get("Raw"), - b"[1, 2, 3]", - headers=Headers(content_type="application/json") - ) - assert "Raw" in r[0] - - r = cv.get_content_view( - cv.get("Auto"), - b"[1, 2, 3]", - headers=Headers(content_type="application/json") - ) - assert r[0] == "JSON" - - r = cv.get_content_view( - cv.get("Auto"), - b"[1, 2", - headers=Headers(content_type="application/json") - ) - assert "Raw" in r[0] - - r = cv.get_content_view( - cv.get("Auto"), - b"[1, 2, 3]", - headers=Headers(content_type="application/vnd.api+json") - ) - assert r[0] == "JSON" - - tutils.raises( - ContentViewException, - cv.get_content_view, - cv.get("AMF"), - b"[1, 2", - headers=Headers() - ) - def test_add_cv(self): class TestContentView(cv.View): name = "test" @@ -231,6 +198,57 @@ Larry ) +def test_get_content_view(): + desc, lines, err = cv.get_content_view( + cv.get("Raw"), + b"[1, 2, 3]", + ) + assert "Raw" in desc + assert list(lines) + assert not err + + desc, lines, err = cv.get_content_view( + cv.get("Auto"), + b"[1, 2, 3]", + headers=Headers(content_type="application/json") + ) + assert desc == "JSON" + + desc, lines, err = cv.get_content_view( + cv.get("JSON"), + b"[1, 2", + ) + assert "Couldn't parse" in desc + + with mock.patch("mitmproxy.contentviews.ViewAuto.__call__") as view_auto: + view_auto.side_effect = ValueError + + desc, lines, err = cv.get_content_view( + cv.get("JSON"), + b"[1, 2", + ) + assert err + assert "Couldn't parse" in desc + + +def test_get_message_content_view(): + r = netlib.tutils.treq() + desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r) + assert desc == "Raw" + + r.encode("gzip") + desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r) + assert desc == "[decoded gzip] Raw" + + r.headers["content-encoding"] = "deflate" + desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r) + assert desc == "[cannot decode] Raw" + + r.content = None + desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r) + assert list(lines) == [[("error", "content missing")]] + + if pyamf: def test_view_amf_request(): v = cv.ViewAMF() @@ -264,4 +282,4 @@ def test_pretty_json(): assert cv.pretty_json(b'{"foo": 1}') assert not cv.pretty_json(b"moo") assert cv.pretty_json(b'{"foo" : "\xe4\xb8\x96\xe7\x95\x8c"}') # utf8 with chinese characters - assert not cv.pretty_json(b'{"foo" : "\xFF"}') + assert not cv.pretty_json(b'{"foo" : "\xFF"}') \ No newline at end of file From fd7b6b958e7da1073b68a13e817ceaacaf6ad28c Mon Sep 17 00:00:00 2001 From: Clemens Date: Wed, 3 Aug 2016 15:40:23 +0200 Subject: [PATCH 11/18] litte style changes --- web/src/css/flowdetail.less | 4 ++++ web/src/js/components/ContentView/ShowFullContentButton.jsx | 2 +- web/src/js/components/common/Button.jsx | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/web/src/css/flowdetail.less b/web/src/css/flowdetail.less index 7e65528fc..b4c7047b9 100644 --- a/web/src/css/flowdetail.less +++ b/web/src/css/flowdetail.less @@ -69,6 +69,10 @@ margin-top: 10px; } +.view-all-content-btn{ + float: right; +} + .flow-detail table { .monospace(); width: 100%; diff --git a/web/src/js/components/ContentView/ShowFullContentButton.jsx b/web/src/js/components/ContentView/ShowFullContentButton.jsx index a0217d32b..17e61be46 100644 --- a/web/src/js/components/ContentView/ShowFullContentButton.jsx +++ b/web/src/js/components/ContentView/ShowFullContentButton.jsx @@ -14,7 +14,7 @@ ShowFullContentButton.propTypes = { function ShowFullContentButton ( {setShowFullContent, showFullContent} ){ return ( - !showFullContent &&